Currently the method of dealing with an IO operation on a bus (PIO/MMIO) is to call the read or write callback for each device registered on the bus until we find a device which handles it.
Since the number of devices on a bus can be significant due to ioeventfds and coalesced MMIO zones, this leads to a lot of overhead on each IO operation. Instead of registering devices, we now register ranges which points to a device. Lookup is done using an efficient bsearch instead of a linear search. This should speed up all IO operations generated by the guest. Cc: Avi Kivity <[email protected]> Cc: Marcelo Tosatti <[email protected]> Signed-off-by: Sasha Levin <[email protected]> --- This patch depends on '[PATCH v3] MMIO: Make coalesced mmio use a device per zone'. arch/x86/kvm/i8254.c | 4 +- arch/x86/kvm/i8259.c | 4 +- include/linux/kvm_host.h | 18 ++++---- virt/kvm/coalesced_mmio.c | 6 +-- virt/kvm/eventfd.c | 3 +- virt/kvm/ioapic.c | 13 +----- virt/kvm/kvm_main.c | 107 +++++++++++++++++++++++++++++++++++++++----- 7 files changed, 115 insertions(+), 40 deletions(-) diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index efad723..61d193c 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c @@ -713,13 +713,15 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags) kvm_register_irq_mask_notifier(kvm, 0, &pit->mask_notifier); kvm_iodevice_init(&pit->dev, &pit_dev_ops); - ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, &pit->dev); + ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, KVM_PIT_BASE_ADDRESS, + KVM_PIT_MEM_LENGTH, &pit->dev); if (ret < 0) goto fail; if (flags & KVM_PIT_SPEAKER_DUMMY) { kvm_iodevice_init(&pit->speaker_dev, &speaker_dev_ops); ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, + KVM_SPEAKER_BASE_ADDRESS, 4, &pit->speaker_dev); if (ret < 0) goto fail_unregister; diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c index 19fe855..c2295af 100644 --- a/arch/x86/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c @@ -562,7 +562,9 @@ struct kvm_pic *kvm_create_pic(struct kvm *kvm) */ kvm_iodevice_init(&s->dev, &picdev_ops); mutex_lock(&kvm->slots_lock); - ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, &s->dev); + ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, 0x20, 2, &s->dev); + ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, 0xa0, 2, &s->dev); + ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, 0x4d0, 2, &s->dev); mutex_unlock(&kvm->slots_lock); if (ret < 0) { kfree(s); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 4766178..512fed4 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -55,16 +55,16 @@ struct kvm; struct kvm_vcpu; extern struct kmem_cache *kvm_vcpu_cache; -/* - * It would be nice to use something smarter than a linear search, TBD... - * Thankfully we dont expect many devices to register (famous last words :), - * so until then it will suffice. At least its abstracted so we can change - * in one place. - */ +struct kvm_io_range { + gpa_t addr; + int len; + struct kvm_io_device *dev; +}; + struct kvm_io_bus { - int dev_count; + int dev_count; #define NR_IOBUS_DEVS 300 - struct kvm_io_device *devs[NR_IOBUS_DEVS]; + struct kvm_io_range range[NR_IOBUS_DEVS]; }; enum kvm_bus { @@ -78,7 +78,7 @@ int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len, void *val); int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, - struct kvm_io_device *dev); + gpa_t addr, int len, struct kvm_io_device *dev); int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, struct kvm_io_device *dev); diff --git a/virt/kvm/coalesced_mmio.c b/virt/kvm/coalesced_mmio.c index c0e37d7..70f69cf 100644 --- a/virt/kvm/coalesced_mmio.c +++ b/virt/kvm/coalesced_mmio.c @@ -60,9 +60,6 @@ static int coalesced_mmio_write(struct kvm_io_device *this, struct kvm_coalesced_mmio_dev *dev = to_mmio(this); struct kvm_coalesced_mmio_ring *ring = dev->kvm->coalesced_mmio_ring; - if (!coalesced_mmio_in_range(dev, addr, len)) - return -EOPNOTSUPP; - spin_lock(&dev->kvm->coalesced_zones.lock); if (!coalesced_mmio_has_room(dev)) { @@ -141,7 +138,8 @@ int kvm_vm_ioctl_register_coalesced_mmio(struct kvm *kvm, dev->zone = *zone; mutex_lock(&kvm->slots_lock); - ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, &dev->dev); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, zone->addr, + zone->size, &dev->dev); mutex_unlock(&kvm->slots_lock); if (ret < 0) goto out_free_dev; diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 73358d2..ab02042 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -586,7 +586,8 @@ kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) kvm_iodevice_init(&p->dev, &ioeventfd_ops); - ret = kvm_io_bus_register_dev(kvm, bus_idx, &p->dev); + ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, + p->length, &p->dev); if (ret < 0) goto unlock_fail; diff --git a/virt/kvm/ioapic.c b/virt/kvm/ioapic.c index 8df1ca1..72559a7 100644 --- a/virt/kvm/ioapic.c +++ b/virt/kvm/ioapic.c @@ -271,19 +271,11 @@ static inline struct kvm_ioapic *to_ioapic(struct kvm_io_device *dev) return container_of(dev, struct kvm_ioapic, dev); } -static inline int ioapic_in_range(struct kvm_ioapic *ioapic, gpa_t addr) -{ - return ((addr >= ioapic->base_address && - (addr < ioapic->base_address + IOAPIC_MEM_LENGTH))); -} - static int ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len, void *val) { struct kvm_ioapic *ioapic = to_ioapic(this); u32 result; - if (!ioapic_in_range(ioapic, addr)) - return -EOPNOTSUPP; ioapic_debug("addr %lx\n", (unsigned long)addr); ASSERT(!(addr & 0xf)); /* check alignment */ @@ -325,8 +317,6 @@ static int ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, { struct kvm_ioapic *ioapic = to_ioapic(this); u32 data; - if (!ioapic_in_range(ioapic, addr)) - return -EOPNOTSUPP; ioapic_debug("ioapic_mmio_write addr=%p len=%d val=%p\n", (void*)addr, len, val); @@ -394,7 +384,8 @@ int kvm_ioapic_init(struct kvm *kvm) kvm_iodevice_init(&ioapic->dev, &ioapic_mmio_ops); ioapic->kvm = kvm; mutex_lock(&kvm->slots_lock); - ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, &ioapic->dev); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ioapic->base_address, + IOAPIC_MEM_LENGTH, &ioapic->dev); mutex_unlock(&kvm->slots_lock); if (ret < 0) { kvm->arch.vioapic = NULL; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index aefdda3..28c6f01 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2391,24 +2391,99 @@ static void kvm_io_bus_destroy(struct kvm_io_bus *bus) int i; for (i = 0; i < bus->dev_count; i++) { - struct kvm_io_device *pos = bus->devs[i]; + struct kvm_io_device *pos = bus->range[i].dev; kvm_iodevice_destructor(pos); } kfree(bus); } +int kvm_io_bus_find_closest_dev_idx(struct kvm_io_bus *bus, + gpa_t addr, int len) +{ + int start = 0, end = bus->dev_count - 1; + + if (bus->dev_count == 0) + return -1; + + while (start <= end) { + int mid = (start + end) / 2; + struct kvm_io_range *range = &bus->range[mid]; + + if (addr > range->addr) + start = mid + 1; + else if (addr < range->addr) + end = mid - 1; + else + return mid; + } + + return start - 1; +} + +struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, + gpa_t addr, int len) +{ + int idx; + + idx = kvm_io_bus_find_closest_dev_idx(bus, addr, len); + if (idx < 0) + return NULL; + + /* Verify that [addr, addr+len] is contained within the range */ + if ((bus->range[idx].addr > addr) || + ((bus->range[idx].addr + bus->range[idx].len) < (addr + len))) + return NULL; + + return bus->range[idx].dev; +} + +int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, + gpa_t addr, int len) +{ + int new_idx, i; + + if (bus->dev_count == NR_IOBUS_DEVS) + return -ENOSPC; + + /* This is where the new entry should be located */ + new_idx = kvm_io_bus_find_closest_dev_idx(bus, addr, len); + + /* This should be the first device on the bus */ + if (new_idx < 0) + new_idx = 0; + /* Place the device after the existing device */ + else + new_idx++; + + /* Clear it by shifting the array to the right at index */ + memmove(&bus->range[new_idx + 1], &bus->range[new_idx], + sizeof(bus->range[new_idx]) * + (bus->dev_count - new_idx)); + + bus->range[new_idx] = (struct kvm_io_range) { + .addr = addr, + .len = len, + .dev = dev, + }; + + bus->dev_count++; + + return 0; +} + /* kvm_io_bus_write - called under kvm->slots_lock */ int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len, const void *val) { - int i; struct kvm_io_bus *bus; + struct kvm_io_device *dev; bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); - for (i = 0; i < bus->dev_count; i++) - if (!kvm_iodevice_write(bus->devs[i], addr, len, val)) - return 0; + dev = kvm_io_bus_find_dev(bus, addr, len); + if (dev && !kvm_iodevice_write(dev, addr, len, val)) + return 0; + return -EOPNOTSUPP; } @@ -2416,19 +2491,20 @@ int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len, void *val) { - int i; struct kvm_io_bus *bus; + struct kvm_io_device *dev; bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); - for (i = 0; i < bus->dev_count; i++) - if (!kvm_iodevice_read(bus->devs[i], addr, len, val)) - return 0; + dev = kvm_io_bus_find_dev(bus, addr, len); + if (dev && !kvm_iodevice_read(dev, addr, len, val)) + return 0; + return -EOPNOTSUPP; } /* Caller must hold slots_lock. */ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, - struct kvm_io_device *dev) + gpa_t addr, int len, struct kvm_io_device *dev) { struct kvm_io_bus *new_bus, *bus; @@ -2440,7 +2516,7 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, if (!new_bus) return -ENOMEM; memcpy(new_bus, bus, sizeof(struct kvm_io_bus)); - new_bus->devs[new_bus->dev_count++] = dev; + kvm_io_bus_insert_dev(new_bus, dev, addr, len); rcu_assign_pointer(kvm->buses[bus_idx], new_bus); synchronize_srcu_expedited(&kvm->srcu); kfree(bus); @@ -2464,9 +2540,14 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, r = -ENOENT; for (i = 0; i < new_bus->dev_count; i++) - if (new_bus->devs[i] == dev) { + if (new_bus->range[i].dev == dev) { r = 0; - new_bus->devs[i] = new_bus->devs[--new_bus->dev_count]; + memmove(&new_bus->range[i], &new_bus->range[i+1], + sizeof(new_bus->range[i]) * + (new_bus->dev_count - i)); + + new_bus->dev_count--; + i--; break; } -- 1.7.6 -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
