Re: [Qemu-devel] [RFC v2 2/3] intc/arm_gic: Support IRQ injection for more than 256 vpus
Hi Drew,
On 9/12/19 9:36 AM, Andrew Jones wrote:
> On Wed, Sep 11, 2019 at 05:51:24PM +0200, Eric Auger wrote:
>> Host kernels that expose the KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 capability
>> allow injection of interrupts along with vcpu ids larger than 255.
>> Let's encode the vpcu id on 12 bits according to the upgraded KVM_IRQ_LINE
>> ABI when needed.
>>
>> Given that we have two callsites that need to assemble
>> the value for kvm_set_irq(), a new helper routine, kvm_arm_set_irq
>> is introduced.
>>
>> Without that patch qemu exits with "kvm_set_irq: Invalid argument"
>> message.
>>
>> Signed-off-by: Eric Auger
>> Reported-by: Zenghui Yu
>> ---
>> hw/intc/arm_gic_kvm.c | 7 ++-
>> target/arm/cpu.c | 10 --
>> target/arm/kvm.c | 16
>> target/arm/kvm_arm.h | 1 +
>> 4 files changed, 23 insertions(+), 11 deletions(-)
>>
>> diff --git a/hw/intc/arm_gic_kvm.c b/hw/intc/arm_gic_kvm.c
>> index b56fda144f..9deb15e7e6 100644
>> --- a/hw/intc/arm_gic_kvm.c
>> +++ b/hw/intc/arm_gic_kvm.c
>> @@ -55,7 +55,7 @@ void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int
>> level)
>> * has separate fields in the irq number for type,
>> * CPU number and interrupt number.
>> */
>> -int kvm_irq, irqtype, cpu;
>> +int irqtype, cpu;
>>
>> if (irq < (num_irq - GIC_INTERNAL)) {
>> /* External interrupt. The kernel numbers these like the GIC
>> @@ -72,10 +72,7 @@ void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int
>> level)
>> cpu = irq / GIC_INTERNAL;
>> irq %= GIC_INTERNAL;
>> }
>> -kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT)
>> -| (cpu << KVM_ARM_IRQ_VCPU_SHIFT) | irq;
>> -
>> -kvm_set_irq(kvm_state, kvm_irq, !!level);
>> +kvm_arm_set_irq(cpu, irqtype, irq, !!level);
>> }
>>
>> static void kvm_arm_gicv2_set_irq(void *opaque, int irq, int level)
>> diff --git a/target/arm/cpu.c b/target/arm/cpu.c
>> index 2399c14471..13813fb213 100644
>> --- a/target/arm/cpu.c
>> +++ b/target/arm/cpu.c
>> @@ -576,16 +576,16 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq,
>> int level)
>> ARMCPU *cpu = opaque;
>> CPUARMState *env = &cpu->env;
>> CPUState *cs = CPU(cpu);
>> -int kvm_irq = KVM_ARM_IRQ_TYPE_CPU << KVM_ARM_IRQ_TYPE_SHIFT;
>> uint32_t linestate_bit;
>> +int irq_id;
>>
>> switch (irq) {
>> case ARM_CPU_IRQ:
>> -kvm_irq |= KVM_ARM_IRQ_CPU_IRQ;
>> +irq_id = KVM_ARM_IRQ_CPU_IRQ;
>> linestate_bit = CPU_INTERRUPT_HARD;
>> break;
>> case ARM_CPU_FIQ:
>> -kvm_irq |= KVM_ARM_IRQ_CPU_FIQ;
>> +irq_id = KVM_ARM_IRQ_CPU_FIQ;
>> linestate_bit = CPU_INTERRUPT_FIQ;
>> break;
>> default:
>> @@ -597,9 +597,7 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq,
>> int level)
>> } else {
>> env->irq_line_state &= ~linestate_bit;
>> }
>> -
>> -kvm_irq |= cs->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
>> -kvm_set_irq(kvm_state, kvm_irq, level ? 1 : 0);
>> +kvm_arm_set_irq(cs->cpu_index, KVM_ARM_IRQ_TYPE_CPU, irq_id, !!level);
>> #endif
>> }
>>
>> diff --git a/target/arm/kvm.c b/target/arm/kvm.c
>> index b2eaa50b8d..6cdfa2204f 100644
>> --- a/target/arm/kvm.c
>> +++ b/target/arm/kvm.c
>> @@ -744,6 +744,22 @@ int kvm_arm_vgic_probe(void)
>> }
>> }
>>
>> +int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
>> +{
>> +int kvm_irq = 0;
>
> No need to init to zero, and could just immediately init with the
> line below instead.
>
>> +
>> +kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
>> +
>> +if (cpu != 0) {
>
> No need for this 'if'
>
>> +int cpu_idx2 = cpu / 256;
>> +int cpu_idx1 = cpu % 256;
>> +
>> +kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
>> + ((cpu_idx2 & KVM_ARM_IRQ_VCPU2_MASK) << KVM_ARM_IRQ_VCPU2_SHIFT);
>
> Masking should be unnecessary as the only way it'll do anything is if we
> have vcpus >= 4096, which I imagine will never happen or will be guarded
> against happening elsewhere. Silently masking doesn't look right anyway,
> so I'd either add an assert(cpu_idx2 < 16) and drop the masking or just
> drop the masking.
All that makes sense.
Thank you for the review.
Eric
>
>> +}
>> +return kvm_set_irq(kvm_state, kvm_irq, !!level);
>> +}
>> +
>> int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
>> uint64_t address, uint32_t data, PCIDevice
>> *dev)
>> {
>> diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
>> index b3106c8600..b4e19457a0 100644
>> --- a/target/arm/kvm_arm.h
>> +++ b/target/arm/kvm_arm.h
>> @@ -253,6 +253,7 @@ int kvm_arm_vgic_probe(void);
>>
>> void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
>> void kvm_arm_pmu_init(CPUState *cs);
>> +int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level);
>>
>> #else
>>
>> --
>> 2.20.1
>>
>>
>
> Thanks,
> drew
>
Re: [Qemu-devel] [RFC v2 2/3] intc/arm_gic: Support IRQ injection for more than 256 vpus
On Wed, Sep 11, 2019 at 05:51:24PM +0200, Eric Auger wrote:
> Host kernels that expose the KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 capability
> allow injection of interrupts along with vcpu ids larger than 255.
> Let's encode the vpcu id on 12 bits according to the upgraded KVM_IRQ_LINE
> ABI when needed.
>
> Given that we have two callsites that need to assemble
> the value for kvm_set_irq(), a new helper routine, kvm_arm_set_irq
> is introduced.
>
> Without that patch qemu exits with "kvm_set_irq: Invalid argument"
> message.
>
> Signed-off-by: Eric Auger
> Reported-by: Zenghui Yu
> ---
> hw/intc/arm_gic_kvm.c | 7 ++-
> target/arm/cpu.c | 10 --
> target/arm/kvm.c | 16
> target/arm/kvm_arm.h | 1 +
> 4 files changed, 23 insertions(+), 11 deletions(-)
>
> diff --git a/hw/intc/arm_gic_kvm.c b/hw/intc/arm_gic_kvm.c
> index b56fda144f..9deb15e7e6 100644
> --- a/hw/intc/arm_gic_kvm.c
> +++ b/hw/intc/arm_gic_kvm.c
> @@ -55,7 +55,7 @@ void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int
> level)
> * has separate fields in the irq number for type,
> * CPU number and interrupt number.
> */
> -int kvm_irq, irqtype, cpu;
> +int irqtype, cpu;
>
> if (irq < (num_irq - GIC_INTERNAL)) {
> /* External interrupt. The kernel numbers these like the GIC
> @@ -72,10 +72,7 @@ void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int
> level)
> cpu = irq / GIC_INTERNAL;
> irq %= GIC_INTERNAL;
> }
> -kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT)
> -| (cpu << KVM_ARM_IRQ_VCPU_SHIFT) | irq;
> -
> -kvm_set_irq(kvm_state, kvm_irq, !!level);
> +kvm_arm_set_irq(cpu, irqtype, irq, !!level);
> }
>
> static void kvm_arm_gicv2_set_irq(void *opaque, int irq, int level)
> diff --git a/target/arm/cpu.c b/target/arm/cpu.c
> index 2399c14471..13813fb213 100644
> --- a/target/arm/cpu.c
> +++ b/target/arm/cpu.c
> @@ -576,16 +576,16 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq,
> int level)
> ARMCPU *cpu = opaque;
> CPUARMState *env = &cpu->env;
> CPUState *cs = CPU(cpu);
> -int kvm_irq = KVM_ARM_IRQ_TYPE_CPU << KVM_ARM_IRQ_TYPE_SHIFT;
> uint32_t linestate_bit;
> +int irq_id;
>
> switch (irq) {
> case ARM_CPU_IRQ:
> -kvm_irq |= KVM_ARM_IRQ_CPU_IRQ;
> +irq_id = KVM_ARM_IRQ_CPU_IRQ;
> linestate_bit = CPU_INTERRUPT_HARD;
> break;
> case ARM_CPU_FIQ:
> -kvm_irq |= KVM_ARM_IRQ_CPU_FIQ;
> +irq_id = KVM_ARM_IRQ_CPU_FIQ;
> linestate_bit = CPU_INTERRUPT_FIQ;
> break;
> default:
> @@ -597,9 +597,7 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq,
> int level)
> } else {
> env->irq_line_state &= ~linestate_bit;
> }
> -
> -kvm_irq |= cs->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
> -kvm_set_irq(kvm_state, kvm_irq, level ? 1 : 0);
> +kvm_arm_set_irq(cs->cpu_index, KVM_ARM_IRQ_TYPE_CPU, irq_id, !!level);
> #endif
> }
>
> diff --git a/target/arm/kvm.c b/target/arm/kvm.c
> index b2eaa50b8d..6cdfa2204f 100644
> --- a/target/arm/kvm.c
> +++ b/target/arm/kvm.c
> @@ -744,6 +744,22 @@ int kvm_arm_vgic_probe(void)
> }
> }
>
> +int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
> +{
> +int kvm_irq = 0;
No need to init to zero, and could just immediately init with the
line below instead.
> +
> +kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
> +
> +if (cpu != 0) {
No need for this 'if'
> +int cpu_idx2 = cpu / 256;
> +int cpu_idx1 = cpu % 256;
> +
> +kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
> + ((cpu_idx2 & KVM_ARM_IRQ_VCPU2_MASK) << KVM_ARM_IRQ_VCPU2_SHIFT);
Masking should be unnecessary as the only way it'll do anything is if we
have vcpus >= 4096, which I imagine will never happen or will be guarded
against happening elsewhere. Silently masking doesn't look right anyway,
so I'd either add an assert(cpu_idx2 < 16) and drop the masking or just
drop the masking.
> +}
> +return kvm_set_irq(kvm_state, kvm_irq, !!level);
> +}
> +
> int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
> uint64_t address, uint32_t data, PCIDevice *dev)
> {
> diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
> index b3106c8600..b4e19457a0 100644
> --- a/target/arm/kvm_arm.h
> +++ b/target/arm/kvm_arm.h
> @@ -253,6 +253,7 @@ int kvm_arm_vgic_probe(void);
>
> void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
> void kvm_arm_pmu_init(CPUState *cs);
> +int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level);
>
> #else
>
> --
> 2.20.1
>
>
Thanks,
drew
[Qemu-devel] [RFC v2 2/3] intc/arm_gic: Support IRQ injection for more than 256 vpus
Host kernels that expose the KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 capability
allow injection of interrupts along with vcpu ids larger than 255.
Let's encode the vpcu id on 12 bits according to the upgraded KVM_IRQ_LINE
ABI when needed.
Given that we have two callsites that need to assemble
the value for kvm_set_irq(), a new helper routine, kvm_arm_set_irq
is introduced.
Without that patch qemu exits with "kvm_set_irq: Invalid argument"
message.
Signed-off-by: Eric Auger
Reported-by: Zenghui Yu
---
hw/intc/arm_gic_kvm.c | 7 ++-
target/arm/cpu.c | 10 --
target/arm/kvm.c | 16
target/arm/kvm_arm.h | 1 +
4 files changed, 23 insertions(+), 11 deletions(-)
diff --git a/hw/intc/arm_gic_kvm.c b/hw/intc/arm_gic_kvm.c
index b56fda144f..9deb15e7e6 100644
--- a/hw/intc/arm_gic_kvm.c
+++ b/hw/intc/arm_gic_kvm.c
@@ -55,7 +55,7 @@ void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level)
* has separate fields in the irq number for type,
* CPU number and interrupt number.
*/
-int kvm_irq, irqtype, cpu;
+int irqtype, cpu;
if (irq < (num_irq - GIC_INTERNAL)) {
/* External interrupt. The kernel numbers these like the GIC
@@ -72,10 +72,7 @@ void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int
level)
cpu = irq / GIC_INTERNAL;
irq %= GIC_INTERNAL;
}
-kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT)
-| (cpu << KVM_ARM_IRQ_VCPU_SHIFT) | irq;
-
-kvm_set_irq(kvm_state, kvm_irq, !!level);
+kvm_arm_set_irq(cpu, irqtype, irq, !!level);
}
static void kvm_arm_gicv2_set_irq(void *opaque, int irq, int level)
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 2399c14471..13813fb213 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -576,16 +576,16 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq,
int level)
ARMCPU *cpu = opaque;
CPUARMState *env = &cpu->env;
CPUState *cs = CPU(cpu);
-int kvm_irq = KVM_ARM_IRQ_TYPE_CPU << KVM_ARM_IRQ_TYPE_SHIFT;
uint32_t linestate_bit;
+int irq_id;
switch (irq) {
case ARM_CPU_IRQ:
-kvm_irq |= KVM_ARM_IRQ_CPU_IRQ;
+irq_id = KVM_ARM_IRQ_CPU_IRQ;
linestate_bit = CPU_INTERRUPT_HARD;
break;
case ARM_CPU_FIQ:
-kvm_irq |= KVM_ARM_IRQ_CPU_FIQ;
+irq_id = KVM_ARM_IRQ_CPU_FIQ;
linestate_bit = CPU_INTERRUPT_FIQ;
break;
default:
@@ -597,9 +597,7 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq, int
level)
} else {
env->irq_line_state &= ~linestate_bit;
}
-
-kvm_irq |= cs->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
-kvm_set_irq(kvm_state, kvm_irq, level ? 1 : 0);
+kvm_arm_set_irq(cs->cpu_index, KVM_ARM_IRQ_TYPE_CPU, irq_id, !!level);
#endif
}
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index b2eaa50b8d..6cdfa2204f 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -744,6 +744,22 @@ int kvm_arm_vgic_probe(void)
}
}
+int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
+{
+int kvm_irq = 0;
+
+kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
+
+if (cpu != 0) {
+int cpu_idx2 = cpu / 256;
+int cpu_idx1 = cpu % 256;
+
+kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
+ ((cpu_idx2 & KVM_ARM_IRQ_VCPU2_MASK) << KVM_ARM_IRQ_VCPU2_SHIFT);
+}
+return kvm_set_irq(kvm_state, kvm_irq, !!level);
+}
+
int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
uint64_t address, uint32_t data, PCIDevice *dev)
{
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index b3106c8600..b4e19457a0 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -253,6 +253,7 @@ int kvm_arm_vgic_probe(void);
void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
void kvm_arm_pmu_init(CPUState *cs);
+int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level);
#else
--
2.20.1
