When GIC wants to inject a virtual interrupt to a VCPU that is not currently scheduled, it sends a kick for a hypervisor to schedule it. To receive such kicks, we need to set up a doorbell interrupt for each VPE.
Add changes necessary to allocate, mask/unmask and handle doorbell interrupts for each VPE. When a doorbell interrupt is received, set the pending_last flag for the corresponding VPE and kick it, so that the hypervisor schedules the VCPU to handle pending VLPIs. Signed-off-by: Mykyta Poturai <[email protected]> --- xen/arch/arm/gic-v3-its.c | 13 ++- xen/arch/arm/gic-v3-lpi.c | 69 ++++++++++---- xen/arch/arm/gic-v4-its.c | 127 ++++++++++++++++++++++++++ xen/arch/arm/include/asm/gic_v3_its.h | 6 +- xen/arch/arm/vgic-v3-its.c | 14 ++- 5 files changed, 203 insertions(+), 26 deletions(-) diff --git a/xen/arch/arm/gic-v3-its.c b/xen/arch/arm/gic-v3-its.c index be840fbc8f..fa5c1eb6d1 100644 --- a/xen/arch/arm/gic-v3-its.c +++ b/xen/arch/arm/gic-v3-its.c @@ -1016,8 +1016,11 @@ int gicv3_its_map_guest_device(struct domain *d, dev->guest_doorbell = guest_doorbell; dev->guest_devid = guest_devid; - dev->host_devid = host_devid; - dev->eventids = nr_events; + + #ifdef CONFIG_GICV4 + spin_lock_init(&dev->event_map.vlpi_lock); + dev->event_map.nr_lpis = nr_events; + #endif rb_link_node(&dev->rbnode, parent, new); rb_insert_color(&dev->rbnode, &d->arch.vgic.its_devices); @@ -1142,7 +1145,8 @@ int gicv3_remove_guest_event(struct domain *d, paddr_t vdoorbell_address, if ( host_lpi == INVALID_LPI ) return -EINVAL; - gicv3_lpi_update_host_entry(host_lpi, d->domain_id, INVALID_LPI); + gicv3_lpi_update_host_entry(host_lpi, d->domain_id, INVALID_LPI, + false, INVALID_VCPU_ID); return 0; } @@ -1169,7 +1173,8 @@ struct pending_irq *gicv3_assign_guest_event(struct domain *d, if ( !pirq ) return NULL; - gicv3_lpi_update_host_entry(host_lpi, d->domain_id, virt_lpi); + gicv3_lpi_update_host_entry(host_lpi, d->domain_id, virt_lpi, + false, INVALID_VCPU_ID); return pirq; } diff --git a/xen/arch/arm/gic-v3-lpi.c b/xen/arch/arm/gic-v3-lpi.c index 3c2649b695..37f1aa1064 100644 --- a/xen/arch/arm/gic-v3-lpi.c +++ b/xen/arch/arm/gic-v3-lpi.c @@ -39,7 +39,7 @@ union host_lpi { struct { uint32_t virt_lpi; uint16_t dom_id; - uint16_t pad; + uint16_t db_vcpu_id; }; }; @@ -161,24 +161,48 @@ void gicv3_do_LPI(unsigned int lpi) * ignore them, as they have no further state and no-one can expect * to see them if they have not been mapped. */ - if ( hlpi.virt_lpi == INVALID_LPI ) + if ( hlpi.virt_lpi == INVALID_LPI && hlpi.db_vcpu_id == INVALID_VCPU_ID ) goto out; d = rcu_lock_domain_by_id(hlpi.dom_id); if ( !d ) goto out; - /* - * TODO: Investigate what to do here for potential interrupt storms. - * As we keep all host LPIs enabled, for disabling LPIs we would need - * to queue a ITS host command, which we avoid so far during a guest's - * runtime. Also re-enabling would trigger a host command upon the - * guest sending a command, which could be an attack vector for - * hogging the host command queue. - * See the thread around here for some background: - * https://lists.xen.org/archives/html/xen-devel/2016-12/msg00003.html - */ - vgic_vcpu_inject_lpi(d, hlpi.virt_lpi); + /* It is a doorbell interrupt. */ + if ( hlpi.db_vcpu_id != INVALID_VCPU_ID ) + { +#ifdef CONFIG_GICV4 + struct vcpu *v = d->vcpu[hlpi.db_vcpu_id]; + + /* We got the message, no need to fire again */ + its_vpe_mask_db(v->arch.vgic.its_vpe); + + /* + * Update the pending_last flag that indicates that VLPIs are pending. + * And the corresponding vcpu is also kicked into action. + */ + v->arch.vgic.its_vpe->pending_last = true; + + vcpu_kick(v); +#else + printk(XENLOG_WARNING + "Doorbell LPI is only suooprted on GICV4\n"); +#endif + } + else + { + /* + * TODO: Investigate what to do here for potential interrupt storms. + * As we keep all host LPIs enabled, for disabling LPIs we would need + * to queue a ITS host command, which we avoid so far during a guest's + * runtime. Also re-enabling would trigger a host command upon the + * guest sending a command, which could be an attack vector for + * hogging the host command queue. + * See the thread around here for some background: + * https://lists.xen.org/archives/html/xen-devel/2016-12/msg00003.html + */ + vgic_vcpu_inject_lpi(d, hlpi.virt_lpi); + } rcu_unlock_domain(d); @@ -187,7 +211,8 @@ out: } void gicv3_lpi_update_host_entry(uint32_t host_lpi, int domain_id, - uint32_t virt_lpi) + uint32_t virt_lpi, bool is_db, + uint16_t db_vcpu_id) { union host_lpi *hlpip, hlpi; @@ -197,8 +222,16 @@ void gicv3_lpi_update_host_entry(uint32_t host_lpi, int domain_id, hlpip = &lpi_data.host_lpis[host_lpi / HOST_LPIS_PER_PAGE][host_lpi % HOST_LPIS_PER_PAGE]; - hlpi.virt_lpi = virt_lpi; - hlpi.dom_id = domain_id; + if ( !is_db ) + { + hlpi.virt_lpi = virt_lpi; + hlpi.dom_id = domain_id; + } + else + { + hlpi.dom_id = domain_id; + hlpi.db_vcpu_id = db_vcpu_id; + } write_u64_atomic(&hlpip->data, hlpi.data); } @@ -595,6 +628,7 @@ int gicv3_allocate_host_lpi_block(struct domain *d, uint32_t *first_lpi) */ hlpi.virt_lpi = INVALID_LPI; hlpi.dom_id = d->domain_id; + hlpi.db_vcpu_id = INVALID_VCPU_ID; write_u64_atomic(&lpi_data.host_lpis[chunk][lpi_idx + i].data, hlpi.data); @@ -602,7 +636,8 @@ int gicv3_allocate_host_lpi_block(struct domain *d, uint32_t *first_lpi) * Enable this host LPI, so we don't have to do this during the * guest's runtime. */ - lpi_data.lpi_property[lpi + i] |= LPI_PROP_ENABLED; + lpi_write_config(lpi_data.lpi_property, lpi + i + LPI_OFFSET, 0xff, + LPI_PROP_ENABLED); } lpi_data.next_free_lpi = lpi + LPI_BLOCK; diff --git a/xen/arch/arm/gic-v4-its.c b/xen/arch/arm/gic-v4-its.c index 175fda7acb..0462976b93 100644 --- a/xen/arch/arm/gic-v4-its.c +++ b/xen/arch/arm/gic-v4-its.c @@ -157,6 +157,9 @@ static int its_send_cmd_vmapp(struct host_its *its, struct its_vpe *vpe, cmd[3] = (vpt_addr & GENMASK(51, 16)) | ((HOST_LPIS_NRBITS - 1) & GENMASK(4, 0)); + /* Default doorbell interrupt */ + cmd[1] |= (uint64_t)vpe->vpe_db_lpi; + out: ret = its_send_command(its, cmd); @@ -296,6 +299,37 @@ static int its_send_cmd_vmovp(struct its_vpe *vpe) return 0; } + +static void its_vpe_send_inv_db(struct its_vpe *vpe) +{ + // struct its_device *dev = vpe_proxy.dev; + // unsigned long flags; + + // spin_lock_irqsave(&vpe_proxy.lock, flags); + // gicv4_vpe_db_proxy_map_locked(vpe); + // its_send_cmd_inv(dev->hw_its, dev->host_devid, vpe->vpe_proxy_event); + // spin_unlock_irqrestore(&vpe_proxy.lock, flags); +} + +static void its_vpe_inv_db(struct its_vpe *vpe) +{ + its_vpe_send_inv_db(vpe); +} + +void its_vpe_mask_db(struct its_vpe *vpe) +{ + /* Only clear enable bit. */ + lpi_write_config(lpi_data.lpi_property, vpe->vpe_db_lpi, LPI_PROP_ENABLED, 0); + its_vpe_inv_db(vpe); +} + +static void its_vpe_unmask_db(struct its_vpe *vpe) +{ + /* Only set enable bit. */ + lpi_write_config(lpi_data.lpi_property, vpe->vpe_db_lpi, 0, LPI_PROP_ENABLED); + its_vpe_inv_db(vpe); +} + static void __init its_vpe_teardown(struct its_vpe *vpe) { unsigned int order; @@ -309,6 +343,8 @@ static void __init its_vpe_teardown(struct its_vpe *vpe) int vgic_v4_its_vm_init(struct domain *d) { unsigned int nr_vcpus = d->max_vcpus; + unsigned int nr_db_lpis, nr_chunks, i = 0; + uint32_t *db_lpi_bases; int ret = -ENOMEM; if ( !gicv3_its_host_has_its() ) @@ -326,9 +362,31 @@ int vgic_v4_its_vm_init(struct domain *d) d->arch.vgic.its_vm->vproptable = lpi_allocate_proptable(); if ( !d->arch.vgic.its_vm->vproptable ) goto fail_vprop; + /* Allocate a doorbell interrupt for each VPE. */ + nr_db_lpis = d->arch.vgic.its_vm->nr_vpes; + nr_chunks = DIV_ROUND_UP(nr_db_lpis, LPI_BLOCK); + db_lpi_bases = xzalloc_array(uint32_t, nr_chunks); + if ( !db_lpi_bases ) + goto fail_db_bases; + + do { + /* Allocate doorbell interrupts in chunks of LPI_BLOCK (=32). */ + ret = gicv3_allocate_host_lpi_block(d, &db_lpi_bases[i]); + if ( ret ) + goto fail_db; + } while ( ++i < nr_chunks ); + + d->arch.vgic.its_vm->db_lpi_bases = db_lpi_bases; + d->arch.vgic.its_vm->nr_db_lpis = nr_db_lpis; return 0; +fail_db: + while ( --i >= 0 ) + gicv3_free_host_lpi_block(d->arch.vgic.its_vm->db_lpi_bases[i]); + xfree(db_lpi_bases); +fail_db_bases: + lpi_free_proptable(d->arch.vgic.its_vm->vproptable); fail_vprop: xfree(d->arch.vgic.its_vm->vpes); fail_vpes: @@ -340,8 +398,13 @@ fail_vprop: void vgic_v4_free_its_vm(struct domain *d) { struct its_vm *its_vm = d->arch.vgic.its_vm; + int nr_chunks = DIV_ROUND_UP(its_vm->nr_db_lpis, LPI_BLOCK); if ( its_vm->vpes ) xfree(its_vm->vpes); + while ( --nr_chunks >= 0 ) + gicv3_free_host_lpi_block(its_vm->db_lpi_bases[nr_chunks]); + if ( its_vm->db_lpi_bases ) + xfree(its_vm->db_lpi_bases); if ( its_vm->vproptable ) lpi_free_proptable(its_vm); } @@ -357,14 +420,29 @@ int vgic_v4_its_vpe_init(struct vcpu *vcpu) return -ENOMEM; its_vm->vpes[vcpuid] = vcpu->arch.vgic.its_vpe; + vcpu->arch.vgic.its_vpe = vcpu->arch.vgic.its_vpe; + vcpu->arch.vgic.its_vpe->vpe_db_lpi = its_vm->db_lpi_bases[vcpuid/32] + (vcpuid % 32); + /* + * Sometimes vlpi gets firstly mapped before associated vpe + * becoming resident, so in case missing the interrupt, we intend to + * enable doorbell at the initialization stage + */ + vcpu->arch.vgic.its_vpe->its_vm = its_vm; + gicv3_lpi_update_host_entry(vcpu->arch.vgic.its_vpe->vpe_db_lpi, + vcpu->domain->domain_id, INVALID_LPI, true, + vcpu->vcpu_id); + + ret = its_vpe_init(vcpu->arch.vgic.its_vpe); if ( ret ) { its_vpe_teardown(vcpu->arch.vgic.its_vpe); return ret; } + its_vpe_unmask_db(vcpu->arch.vgic.its_vpe); + return 0; } @@ -800,6 +878,7 @@ void vgic_v4_load(struct vcpu *vcpu) * corresponding to our current CPU expects us here */ WARN_ON(gicv4_vpe_set_affinity(vcpu)); + its_vpe_mask_db(vpe); its_make_vpe_resident(vpe, vcpu->processor); vpe->resident = true; } @@ -812,5 +891,53 @@ void vgic_v4_put(struct vcpu *vcpu, bool need_db) return; its_make_vpe_non_resident(vpe, vcpu->processor); + if ( need_db ) + /* Enable the doorbell, as the guest is going to block */ + its_vpe_unmask_db(vpe); vpe->resident = false; } + +static int its_vlpi_set_doorbell(struct its_vlpi_map *map, bool enable) +{ + if (map->db_enabled == enable) + return 0; + + map->db_enabled = enable; + + /* + * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI + * value or to 1023, depending on the enable bit. But that + * would be issuing a mapping for an /existing/ DevID+EventID + * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI + * to the /same/ vPE, using this opportunity to adjust the doorbell. + */ + return its_send_cmd_vmovi(map->dev->hw_its, map); +} + +int its_vlpi_prop_update(struct pending_irq *pirq, uint8_t property, + bool needs_inv) +{ + struct its_vlpi_map *map; + unsigned int cpu; + int ret; + + if ( !pirq->vlpi_map ) + return -EINVAL; + + map = pirq->vlpi_map; + + /* Cache the updated property and update the vproptable. */ + map->properties = property; + lpi_write_config(map->vm->vproptable, pirq->irq, 0xff, property); + + if ( needs_inv ) + { + cpu = map->vm->vpes[map->vpe_idx]->col_idx; + ret = its_inv_lpi(map->dev->hw_its, map->dev, map->eventid, cpu); + if ( ret ) + return ret; + } + + return its_vlpi_set_doorbell(map, property & LPI_PROP_ENABLED); +} + diff --git a/xen/arch/arm/include/asm/gic_v3_its.h b/xen/arch/arm/include/asm/gic_v3_its.h index f03a8fad47..dababe97cd 100644 --- a/xen/arch/arm/include/asm/gic_v3_its.h +++ b/xen/arch/arm/include/asm/gic_v3_its.h @@ -295,7 +295,9 @@ struct pending_irq *gicv3_assign_guest_event(struct domain *d, uint32_t vdevid, uint32_t eventid, uint32_t virt_lpi); void gicv3_lpi_update_host_entry(uint32_t host_lpi, int domain_id, - uint32_t virt_lpi); + uint32_t virt_lpi, bool is_db, + uint16_t db_vcpu_id); + struct its_baser *its_get_baser(struct host_its *hw_its, uint32_t type); bool its_alloc_table_entry(struct its_baser *baser, uint32_t id); struct page_info *lpi_allocate_pendtable(void); @@ -322,6 +324,8 @@ bool event_is_forwarded_to_vcpu(struct its_device *dev, uint32_t eventid); void its_vpe_mask_db(struct its_vpe *vpe); #endif int gicv4_its_vlpi_unmap(struct pending_irq *pirq); +int its_vlpi_prop_update(struct pending_irq *pirq, uint8_t property, + bool needs_inv); /* ITS quirks handling. */ uint64_t gicv3_its_get_cacheability(void); diff --git a/xen/arch/arm/vgic-v3-its.c b/xen/arch/arm/vgic-v3-its.c index 94f7dd7d90..0a740ad68f 100644 --- a/xen/arch/arm/vgic-v3-its.c +++ b/xen/arch/arm/vgic-v3-its.c @@ -387,7 +387,7 @@ out_unlock: * property table and update the virtual IRQ's state in the given pending_irq. * Must be called with the respective VGIC VCPU lock held. */ -static int update_lpi_property(struct domain *d, struct pending_irq *p) +int update_lpi_property(struct domain *d, struct pending_irq *p, bool needs_inv) { paddr_t addr; uint8_t property; @@ -417,6 +417,9 @@ static int update_lpi_property(struct domain *d, struct pending_irq *p) else clear_bit(GIC_IRQ_GUEST_ENABLED, &p->status); + if ( pirq_is_tied_to_hw(p) ) + return its_vlpi_prop_update(p, property, needs_inv); + return 0; } @@ -430,6 +433,9 @@ static int update_lpi_property(struct domain *d, struct pending_irq *p) */ static void update_lpi_vgic_status(struct vcpu *v, struct pending_irq *p) { + if ( pirq_is_tied_to_hw(p) ) + return; + ASSERT(spin_is_locked(&v->arch.vgic.lock)); if ( test_bit(GIC_IRQ_GUEST_ENABLED, &p->status) ) @@ -479,7 +485,7 @@ static int its_handle_inv(struct virt_its *its, uint64_t *cmdptr) spin_lock_irqsave(&vcpu->arch.vgic.lock, flags); /* Read the property table and update our cached status. */ - if ( update_lpi_property(d, p) ) + if ( update_lpi_property(d, p, true) ) goto out_unlock; /* Check whether the LPI needs to go on a VCPU. */ @@ -552,7 +558,7 @@ static int its_handle_invall(struct virt_its *its, uint64_t *cmdptr) vlpi = pirqs[i]->irq; /* If that fails for a single LPI, carry on to handle the rest. */ - err = update_lpi_property(its->d, pirqs[i]); + err = update_lpi_property(its->d, pirqs[i], false); if ( !err ) update_lpi_vgic_status(vcpu, pirqs[i]); else @@ -785,7 +791,7 @@ static int its_handle_mapti(struct virt_its *its, uint64_t *cmdptr) * We don't need the VGIC VCPU lock here, because the pending_irq isn't * in the radix tree yet. */ - ret = update_lpi_property(its->d, pirq); + ret = update_lpi_property(its->d, pirq, true); if ( ret ) goto out_remove_host_entry; -- 2.51.2
