Re: [PATCH] hw/riscv/virt-acpi-build.c: Add SRAT and SLIT ACPI tables
++Alistair.
Sorry for that! It seems some typo or paste issue occurred when
pushing the patch.
Hi Alistair,
Could you help review this patch?
Thanks,
Haibo
On Mon, Jan 29, 2024 at 7:37 PM Haibo Xu wrote:
>
> On Mon, Jan 29, 2024 at 5:47 PM Andrew Jones wrote:
> >
> > On Mon, Jan 29, 2024 at 05:42:00PM +0800, Haibo Xu wrote:
> > > Enable ACPI NUMA support by adding the following 2 ACPI tables:
> > > SRAT: provides the association for memory/Harts and Proximity Domains
> > > SLIT: provides the relative distance between Proximity Domains
> > >
> > > The SRAT RINTC Affinity Structure definition[1] was based on the recently
> > > approved ACPI CodeFirst ECR[2].
> > >
> > > [1] https://github.com/riscv-non-isa/riscv-acpi/issues/25
> > > [2] https://mantis.uefi.org/mantis/view.php?id=2433
> > >
> > > Signed-off-by: Haibo Xu
> > > ---
> > > hw/riscv/virt-acpi-build.c | 60 ++
> > > 1 file changed, 60 insertions(+)
> > >
> > > diff --git a/hw/riscv/virt-acpi-build.c b/hw/riscv/virt-acpi-build.c
> > > index 26c7e4482d..f0a6b61747 100644
> > > --- a/hw/riscv/virt-acpi-build.c
> > > +++ b/hw/riscv/virt-acpi-build.c
> > > @@ -528,11 +528,61 @@ static void build_madt(GArray *table_data,
> > > acpi_table_end(linker, );
> > > }
> > >
> > > +/*
> > > + * ACPI spec, Revision 6.5+
> > > + * 5.2.16 System Resource Affinity Table (SRAT)
> > > + * REF: https://github.com/riscv-non-isa/riscv-acpi/issues/25
> > > + *
> > > https://drive.google.com/file/d/1YTdDx2IPm5IeZjAW932EYU-tUtgS08tX/view
> > > + */
> > > +static void
> > > +build_srat(GArray *table_data, BIOSLinker *linker, RISCVVirtState *vms)
> > > +{
> > > +int i;
> > > +uint64_t mem_base;
> > > +MachineClass *mc = MACHINE_GET_CLASS(vms);
> > > +MachineState *ms = MACHINE(vms);
> > > +const CPUArchIdList *cpu_list = mc->possible_cpu_arch_ids(ms);
> > > +AcpiTable table = { .sig = "SRAT", .rev = 3, .oem_id = vms->oem_id,
> > > +.oem_table_id = vms->oem_table_id };
> > > +
> > > +acpi_table_begin(, table_data);
> > > +build_append_int_noprefix(table_data, 1, 4); /* Reserved */
> > > +build_append_int_noprefix(table_data, 0, 8); /* Reserved */
> > > +
> > > +for (i = 0; i < cpu_list->len; ++i) {
> > > +uint32_t nodeid = cpu_list->cpus[i].props.node_id;
> > > +/*
> > > + * 5.2.16.8 RINTC Affinity Structure
> > > + */
> > > +build_append_int_noprefix(table_data, 7, 1); /* Type */
> > > +build_append_int_noprefix(table_data, 20, 1); /* Length */
> > > +build_append_int_noprefix(table_data, 0, 2);/* Reserved
> > > */
> > > +build_append_int_noprefix(table_data, nodeid, 4); /* Proximity
> > > Domain */
> > > +build_append_int_noprefix(table_data, i, 4); /* ACPI Processor
> > > UID */
> > > +/* Flags, Table 5-70 */
> > > +build_append_int_noprefix(table_data, 1 /* Flags: Enabled */, 4);
> > > +build_append_int_noprefix(table_data, 0, 4); /* Clock Domain */
> > > +}
> > > +
> > > +mem_base = vms->memmap[VIRT_DRAM].base;
> > > +for (i = 0; i < ms->numa_state->num_nodes; ++i) {
> > > +if (ms->numa_state->nodes[i].node_mem > 0) {
> > > +build_srat_memory(table_data, mem_base,
> > > + ms->numa_state->nodes[i].node_mem, i,
> > > + MEM_AFFINITY_ENABLED);
> > > +mem_base += ms->numa_state->nodes[i].node_mem;
> > > +}
> > > +}
> > > +
> > > +acpi_table_end(linker, );
> > > +}
> > > +
> > > static void virt_acpi_build(RISCVVirtState *s, AcpiBuildTables *tables)
> > > {
> > > GArray *table_offsets;
> > > unsigned dsdt, xsdt;
> > > GArray *tables_blob = tables->table_data;
> > > +MachineState *ms = MACHINE(s);
> > >
> > > table_offsets = g_array_new(false, true,
> > > sizeof(uint32_t));
> > > @@ -565,6 +615,16 @@ static void virt_acpi_build(RISCVVirtState *s,
> > > AcpiBuildTables *tables)
> > > s->oem_table_id);
> > > }
> > >
> > > +if (ms->numa_state->num_nodes > 0) {
> > > +acpi_add_table(table_offsets, tables_blob);
> > > +build_srat(tables_blob, tables->linker, s);
> > > +if (ms->numa_state->have_numa_distance) {
> > > +acpi_add_table(table_offsets, tables_blob);
> > > +build_slit(tables_blob, tables->linker, ms, s->oem_id,
> > > + s->oem_table_id);
> > > +}
> > > +}
> > > +
> > > /* XSDT is pointed to by RSDP */
> > > xsdt = tables_blob->len;
> > > build_xsdt(tables_blob, tables->linker, table_offsets, s->oem_id,
> > > --
> > > 2.34.1
> > >
> > >
> >
> > Reviewed-by: Andrew Jones
>
> Thanks for the review, Andrew!
Re: [PATCH] hw/riscv/virt-acpi-build.c: Add SRAT and SLIT ACPI tables
On Mon, Jan 29, 2024 at 5:47 PM Andrew Jones wrote:
>
> On Mon, Jan 29, 2024 at 05:42:00PM +0800, Haibo Xu wrote:
> > Enable ACPI NUMA support by adding the following 2 ACPI tables:
> > SRAT: provides the association for memory/Harts and Proximity Domains
> > SLIT: provides the relative distance between Proximity Domains
> >
> > The SRAT RINTC Affinity Structure definition[1] was based on the recently
> > approved ACPI CodeFirst ECR[2].
> >
> > [1] https://github.com/riscv-non-isa/riscv-acpi/issues/25
> > [2] https://mantis.uefi.org/mantis/view.php?id=2433
> >
> > Signed-off-by: Haibo Xu
> > ---
> > hw/riscv/virt-acpi-build.c | 60 ++
> > 1 file changed, 60 insertions(+)
> >
> > diff --git a/hw/riscv/virt-acpi-build.c b/hw/riscv/virt-acpi-build.c
> > index 26c7e4482d..f0a6b61747 100644
> > --- a/hw/riscv/virt-acpi-build.c
> > +++ b/hw/riscv/virt-acpi-build.c
> > @@ -528,11 +528,61 @@ static void build_madt(GArray *table_data,
> > acpi_table_end(linker, );
> > }
> >
> > +/*
> > + * ACPI spec, Revision 6.5+
> > + * 5.2.16 System Resource Affinity Table (SRAT)
> > + * REF: https://github.com/riscv-non-isa/riscv-acpi/issues/25
> > + *
> > https://drive.google.com/file/d/1YTdDx2IPm5IeZjAW932EYU-tUtgS08tX/view
> > + */
> > +static void
> > +build_srat(GArray *table_data, BIOSLinker *linker, RISCVVirtState *vms)
> > +{
> > +int i;
> > +uint64_t mem_base;
> > +MachineClass *mc = MACHINE_GET_CLASS(vms);
> > +MachineState *ms = MACHINE(vms);
> > +const CPUArchIdList *cpu_list = mc->possible_cpu_arch_ids(ms);
> > +AcpiTable table = { .sig = "SRAT", .rev = 3, .oem_id = vms->oem_id,
> > +.oem_table_id = vms->oem_table_id };
> > +
> > +acpi_table_begin(, table_data);
> > +build_append_int_noprefix(table_data, 1, 4); /* Reserved */
> > +build_append_int_noprefix(table_data, 0, 8); /* Reserved */
> > +
> > +for (i = 0; i < cpu_list->len; ++i) {
> > +uint32_t nodeid = cpu_list->cpus[i].props.node_id;
> > +/*
> > + * 5.2.16.8 RINTC Affinity Structure
> > + */
> > +build_append_int_noprefix(table_data, 7, 1); /* Type */
> > +build_append_int_noprefix(table_data, 20, 1); /* Length */
> > +build_append_int_noprefix(table_data, 0, 2);/* Reserved */
> > +build_append_int_noprefix(table_data, nodeid, 4); /* Proximity
> > Domain */
> > +build_append_int_noprefix(table_data, i, 4); /* ACPI Processor UID
> > */
> > +/* Flags, Table 5-70 */
> > +build_append_int_noprefix(table_data, 1 /* Flags: Enabled */, 4);
> > +build_append_int_noprefix(table_data, 0, 4); /* Clock Domain */
> > +}
> > +
> > +mem_base = vms->memmap[VIRT_DRAM].base;
> > +for (i = 0; i < ms->numa_state->num_nodes; ++i) {
> > +if (ms->numa_state->nodes[i].node_mem > 0) {
> > +build_srat_memory(table_data, mem_base,
> > + ms->numa_state->nodes[i].node_mem, i,
> > + MEM_AFFINITY_ENABLED);
> > +mem_base += ms->numa_state->nodes[i].node_mem;
> > +}
> > +}
> > +
> > +acpi_table_end(linker, );
> > +}
> > +
> > static void virt_acpi_build(RISCVVirtState *s, AcpiBuildTables *tables)
> > {
> > GArray *table_offsets;
> > unsigned dsdt, xsdt;
> > GArray *tables_blob = tables->table_data;
> > +MachineState *ms = MACHINE(s);
> >
> > table_offsets = g_array_new(false, true,
> > sizeof(uint32_t));
> > @@ -565,6 +615,16 @@ static void virt_acpi_build(RISCVVirtState *s,
> > AcpiBuildTables *tables)
> > s->oem_table_id);
> > }
> >
> > +if (ms->numa_state->num_nodes > 0) {
> > +acpi_add_table(table_offsets, tables_blob);
> > +build_srat(tables_blob, tables->linker, s);
> > +if (ms->numa_state->have_numa_distance) {
> > +acpi_add_table(table_offsets, tables_blob);
> > +build_slit(tables_blob, tables->linker, ms, s->oem_id,
> > + s->oem_table_id);
> > +}
> > +}
> > +
> > /* XSDT is pointed to by RSDP */
> > xsdt = tables_blob->len;
> > build_xsdt(tables_blob, tables->linker, table_offsets, s->oem_id,
> > --
> > 2.34.1
> >
> >
>
> Reviewed-by: Andrew Jones
Thanks for the review, Andrew!
[PATCH] hw/riscv/virt-acpi-build.c: Add SRAT and SLIT ACPI tables
Enable ACPI NUMA support by adding the following 2 ACPI tables:
SRAT: provides the association for memory/Harts and Proximity Domains
SLIT: provides the relative distance between Proximity Domains
The SRAT RINTC Affinity Structure definition[1] was based on the recently
approved ACPI CodeFirst ECR[2].
[1] https://github.com/riscv-non-isa/riscv-acpi/issues/25
[2] https://mantis.uefi.org/mantis/view.php?id=2433
Signed-off-by: Haibo Xu
---
hw/riscv/virt-acpi-build.c | 60 ++
1 file changed, 60 insertions(+)
diff --git a/hw/riscv/virt-acpi-build.c b/hw/riscv/virt-acpi-build.c
index 26c7e4482d..f0a6b61747 100644
--- a/hw/riscv/virt-acpi-build.c
+++ b/hw/riscv/virt-acpi-build.c
@@ -528,11 +528,61 @@ static void build_madt(GArray *table_data,
acpi_table_end(linker, );
}
+/*
+ * ACPI spec, Revision 6.5+
+ * 5.2.16 System Resource Affinity Table (SRAT)
+ * REF: https://github.com/riscv-non-isa/riscv-acpi/issues/25
+ * https://drive.google.com/file/d/1YTdDx2IPm5IeZjAW932EYU-tUtgS08tX/view
+ */
+static void
+build_srat(GArray *table_data, BIOSLinker *linker, RISCVVirtState *vms)
+{
+int i;
+uint64_t mem_base;
+MachineClass *mc = MACHINE_GET_CLASS(vms);
+MachineState *ms = MACHINE(vms);
+const CPUArchIdList *cpu_list = mc->possible_cpu_arch_ids(ms);
+AcpiTable table = { .sig = "SRAT", .rev = 3, .oem_id = vms->oem_id,
+.oem_table_id = vms->oem_table_id };
+
+acpi_table_begin(, table_data);
+build_append_int_noprefix(table_data, 1, 4); /* Reserved */
+build_append_int_noprefix(table_data, 0, 8); /* Reserved */
+
+for (i = 0; i < cpu_list->len; ++i) {
+uint32_t nodeid = cpu_list->cpus[i].props.node_id;
+/*
+ * 5.2.16.8 RINTC Affinity Structure
+ */
+build_append_int_noprefix(table_data, 7, 1); /* Type */
+build_append_int_noprefix(table_data, 20, 1); /* Length */
+build_append_int_noprefix(table_data, 0, 2);/* Reserved */
+build_append_int_noprefix(table_data, nodeid, 4); /* Proximity Domain
*/
+build_append_int_noprefix(table_data, i, 4); /* ACPI Processor UID */
+/* Flags, Table 5-70 */
+build_append_int_noprefix(table_data, 1 /* Flags: Enabled */, 4);
+build_append_int_noprefix(table_data, 0, 4); /* Clock Domain */
+}
+
+mem_base = vms->memmap[VIRT_DRAM].base;
+for (i = 0; i < ms->numa_state->num_nodes; ++i) {
+if (ms->numa_state->nodes[i].node_mem > 0) {
+build_srat_memory(table_data, mem_base,
+ ms->numa_state->nodes[i].node_mem, i,
+ MEM_AFFINITY_ENABLED);
+mem_base += ms->numa_state->nodes[i].node_mem;
+}
+}
+
+acpi_table_end(linker, );
+}
+
static void virt_acpi_build(RISCVVirtState *s, AcpiBuildTables *tables)
{
GArray *table_offsets;
unsigned dsdt, xsdt;
GArray *tables_blob = tables->table_data;
+MachineState *ms = MACHINE(s);
table_offsets = g_array_new(false, true,
sizeof(uint32_t));
@@ -565,6 +615,16 @@ static void virt_acpi_build(RISCVVirtState *s,
AcpiBuildTables *tables)
s->oem_table_id);
}
+if (ms->numa_state->num_nodes > 0) {
+acpi_add_table(table_offsets, tables_blob);
+build_srat(tables_blob, tables->linker, s);
+if (ms->numa_state->have_numa_distance) {
+acpi_add_table(table_offsets, tables_blob);
+build_slit(tables_blob, tables->linker, ms, s->oem_id,
+ s->oem_table_id);
+}
+}
+
/* XSDT is pointed to by RSDP */
xsdt = tables_blob->len;
build_xsdt(tables_blob, tables->linker, table_offsets, s->oem_id,
--
2.34.1
Re: [RFC PATCH v2 4/5] Add migration support for KVM guest with MTE
On Fri, 26 Mar 2021 at 03:38, Dr. David Alan Gilbert
wrote:
>
> * Haibo Xu (haibo...@linaro.org) wrote:
> > On Thu, 18 Mar 2021 at 04:11, Dr. David Alan Gilbert
> > wrote:
> > >
> > > * Haibo Xu (haibo...@linaro.org) wrote:
> > > > To make it easier to keep the page tags sync with
> > > > the page data, tags for one page are appended to
> > > > the data during ram save iteration.
> > > >
> > > > This patch only add the pre-copy migration support.
> > > > Post-copy and compress as well as zero page saving
> > > > are not supported yet.
> > > >
> > > > Signed-off-by: Haibo Xu
> > >
> > > My guess is that this doesn't work with a lot of other options; e.g.
> > > postcopy and probably compression and a bunch of other things.
> > > Postcopy I can see you'll need some interesting kernel changes for -
> > > you'd need to be able to atomically place a page with it's tag data.
> > >
> > > You probably need to add stuff to migrate_caps_check to disable
> > > features that you don't support.
> > >
> >
> > Hi David,
> >
> > Thanks so much for the comments!
> >
> > You are right, this RFC patch only supports pre-copy mode, no
> > postcopy, no compression.
> > As a RFC, here just want to finalize the tag migration process, that is:
> > 1. let the tag go with the page data(the current choice) which may be
> > a little complex to put
> > them into the current migration process.
>
> I think it's probably the easiest.
>
Agree!
> > 2. migrate them separately which is easy to implement with the current
> > migration(treat the tags
> > as device status), but it would be hard to keep the page data and
> > tag to sync with each other.
> > 3. Any other ways?
> >
> > Once the tag migration process is finalized, a new formal patch series
> > with postcopy as well as
> > compression should be reworked.
>
> It's probably best to look what you would have to do to fit it in with
> those before finalising the basic version; not necessary implement them
> - just get an idea.
>
Thanks for the suggestion!
> > What's more, you mentioned that "some interesting kernel changes are
> > needed to atomically
> > place a page with it's tag data". You mean a single kernel API to
> > store page data and tag in
> > the migration load process?
>
> Yes; see migration/postcopy-ram.c postcopy_place_page - it has the
> problem of placing the whole page in memory before the guest starts
> usign it; I think for MTE you're going to have to put the MTE data in at
> the same time.
>
Good point! Will check this with the kernel guys.
> Dave
>
> >
> > Regards,
> > Haibo
> >
> > > > ---
> > > > include/hw/arm/virt.h| 2 +
> > > > include/migration/misc.h | 1 +
> > > > migration/ram.c | 86 +++-
> > > > 3 files changed, 88 insertions(+), 1 deletion(-)
> > > >
> > > > diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
> > > > index 921416f918..8b28cde8bf 100644
> > > > --- a/include/hw/arm/virt.h
> > > > +++ b/include/hw/arm/virt.h
> > > > @@ -166,6 +166,8 @@ struct VirtMachineState {
> > > > PCIBus *bus;
> > > > char *oem_id;
> > > > char *oem_table_id;
> > > > +/* migrate memory tags */
> > > > +NotifierWithReturn precopy_notifier;
> > > > };
> > > >
> > > > #define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM :
> > > > VIRT_PCIE_ECAM)
> > > > diff --git a/include/migration/misc.h b/include/migration/misc.h
> > > > index bccc1b6b44..005133f471 100644
> > > > --- a/include/migration/misc.h
> > > > +++ b/include/migration/misc.h
> > > > @@ -38,6 +38,7 @@ void precopy_add_notifier(NotifierWithReturn *n);
> > > > void precopy_remove_notifier(NotifierWithReturn *n);
> > > > int precopy_notify(PrecopyNotifyReason reason, Error **errp);
> > > > void precopy_enable_free_page_optimization(void);
> > > > +void precopy_enable_metadata_migration(void);
> > > >
> > > > void ram_mig_init(void);
> > > > void qemu_guest_free_page_hint(void *addr, size_t len);
> > > > diff --git a/migration/ram.c b/migration/ram.c
> > > > index 72143da0ac..e67b798c3b 100644
> > > > --- a/migration/ram.c
> > >
Re: [RFC PATCH v2 4/5] Add migration support for KVM guest with MTE
On Thu, 25 Mar 2021 at 23:37, Juan Quintela wrote:
>
> Haibo Xu wrote:
> > To make it easier to keep the page tags sync with
> > the page data, tags for one page are appended to
> > the data during ram save iteration.
> >
> > This patch only add the pre-copy migration support.
> > Post-copy and compress as well as zero page saving
> > are not supported yet.
> >
> > Signed-off-by: Haibo Xu
>
>
> > #define RAM_SAVE_FLAG_XBZRLE 0x40
> > /* 0x80 is reserved in migration.h start with 0x100 next */
> > #define RAM_SAVE_FLAG_COMPRESS_PAGE0x100
> > +#define RAM_SAVE_FLAG_MTE 0x200
>
> Flags are really a scarce resource. You are using one here, when you
> know that you will always have the feature enable (or not), so you can
> do better during negotiation IMHO.
>
Yes, any suggestions are welcomed to finalize the MTE migration support!
>
> > +void precopy_enable_metadata_migration(void)
> > +{
> > +if (!ram_state) {
> > +return;
> > +}
> > +
> > +ram_state->metadata_enabled = true;
> > +}
>
> My understanding is that in your following patch, if mte is enabled, you
> will always sent mte tags, for all pages needed, right?
Yes, for the current kernel support, we can't tell whether the MTE was enabled
for a specific page.
>
> > +static int save_normal_page_mte_tags(QEMUFile *f, uint8_t *addr)
> > +{
> > +uint8_t *tag_buf = NULL;
> > +uint64_t ipa;
> > +int size = TARGET_PAGE_SIZE / MTE_GRANULE_SIZE;
> > +
> > +if (kvm_physical_memory_addr_from_host(kvm_state, addr, )) {
> > +/* Buffer for the page tags(one byte per tag) */
> > +tag_buf = g_try_malloc0(size);
>
> size of the buffer is known at start of migration. Just get a buffer
> and reuse it?
Yes, we can pre-allocate the buffer during the migration setup time
>
> Do zero pages have mte tags? From migration point of view, a zero page
> is a page that is just full of zeros, i.e. nothing else special.
> Because you are not sending any for them.
>
Yes, I think we can do some optimization for the zero page migration support.
>
>
> > @@ -1148,6 +1219,10 @@ static bool control_save_page(RAMState *rs, RAMBlock
> > *block, ram_addr_t offset,
> > static int save_normal_page(RAMState *rs, RAMBlock *block, ram_addr_t
> > offset,
> > uint8_t *buf, bool async)
> > {
> > +if (rs->metadata_enabled) {
> > +offset |= RAM_SAVE_FLAG_MTE;
>
> You don't really need the flag, for you normal pages are just
> TARGET_PAGE_SIZE + (TARGET_PAGE_SIZE/MTE_)
>
So you suggest to use the size field to indicate whether tags are available?
>
> > +}
> > +
> > ram_counters.transferred += save_page_header(rs, rs->f, block,
> > offset |
> > RAM_SAVE_FLAG_PAGE);
> > if (async) {
> > @@ -1159,6 +1234,11 @@ static int save_normal_page(RAMState *rs, RAMBlock
> > *block, ram_addr_t offset,
> > }
> > ram_counters.transferred += TARGET_PAGE_SIZE;
> > ram_counters.normal++;
> > +
> > +if (rs->metadata_enabled) {
>
> See? You are not checking the flag, you are checking the bool setup at
> the beggining of migration.
The idea is to only migrate the memory tags when MTE was enabled for the VM.
Could you be more elaborate on "You are not checking the flag"?
>
> > +ram_counters.transferred += save_normal_page_mte_tags(rs->f, buf);
> > +}
> > +
> > return 1;
> > }
> >
> > @@ -2189,6 +2269,7 @@ static void ram_state_reset(RAMState *rs)
> > rs->last_version = ram_list.version;
> > rs->ram_bulk_stage = true;
> > rs->fpo_enabled = false;
> > +rs->metadata_enabled = false;
> > }
> >
> > #define MAX_WAIT 50 /* ms, half buffered_file limit */
> > @@ -3779,7 +3860,7 @@ static int ram_load_precopy(QEMUFile *f)
> > trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
> > }
> >
> > -switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
> > +switch (flags & ~(RAM_SAVE_FLAG_CONTINUE | RAM_SAVE_FLAG_MTE)) {
>
> Creating the flag is hurting you here also.
>
> > case RAM_SAVE_FLAG_MEM_SIZE:
> > /* Synchronize RAM block list */
> > total_ram_bytes = addr;
> > @@ -3849,6 +3930,9 @@ static int ram_load_precopy(QEMUFile *f)
> >
> > case RAM_SAVE_FLAG_PAGE:
> > qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
> > +if (flags & RAM_SAVE_FLAG_MTE) {
> > +load_normal_page_mte_tags(f, host);
> > +}
>
> I don't claim to understand the MTE, but my understanding is that if we
> are using MTE, all pages have to have MTE flags, right?
>
> So, somtehing like
>
> is_mte_enabled()
>
> that I told in the other thread looks like a good idea.
>
Yes, we do need a function like this.
> Later, Juan.
>
> > break;
> >
> > case RAM_SAVE_FLAG_COMPRESS_PAGE:
>
Re: [RFC PATCH v2 3/5] Add APIs to get/set MTE tags
On Thu, 25 Mar 2021 at 20:18, Juan Quintela wrote:
>
> Haibo Xu wrote:
> > MTE spec provide instructions to retrieve the memory tags:
> > (1) LDG, at 16 bytes granularity, and available in both user
> > and kernel space;
> > (2) LDGM, at 256 bytes granularity in maximum, and only
> > available in kernel space
> >
> > To improve the performance, KVM has exposed the LDGM capability
> > to user space by providing a new APIs. This patch is just a
> > wrapper for the KVM APIs.
> >
> > Signed-off-by: Haibo Xu
> > ---
> > target/arm/kvm64.c | 24
> > target/arm/kvm_arm.h | 2 ++
> > 2 files changed, 26 insertions(+)
> >
> > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> > index 73a191f8e1..3157025316 100644
> > --- a/target/arm/kvm64.c
> > +++ b/target/arm/kvm64.c
> > @@ -1606,3 +1606,27 @@ bool kvm_arm_verify_ext_dabt_pending(CPUState *cs)
> > }
> > return false;
> > }
> > +
> > +int kvm_arm_mte_get_tags(uint64_t ipa, uint64_t len, uint8_t *buf)
> > +{
> > +struct kvm_arm_copy_mte_tags args = {
> > +.guest_ipa = ipa,
> > +.length = len,
> > +.addr = buf,
> > +.flags = KVM_ARM_TAGS_FROM_GUEST,
> > +};
> > +
> > +return kvm_vm_ioctl(kvm_state, KVM_ARM_MTE_COPY_TAGS, );
>
> Just a question, how fast/slow are this calls?
>
There is no performance data for this API yet, but at least it's more
efficient than that
to only be able to access a single tag by the EL0 "LDG" instruction.
We will try to
collect some performance data for this KVM API later.
> My understanding is that we are making a kvm call for each page that we
> want to migrate, right?
>
Yes, currently I chose to append the tag values to the page data
during the migration.
> Each time that we want to send it.
>
> Later, Juan.
>
>
> > +}
> > +
> > +int kvm_arm_mte_set_tags(uint64_t ipa, uint64_t len, uint8_t *buf)
> > +{
> > +struct kvm_arm_copy_mte_tags args = {
> > +.guest_ipa = ipa,
> > +.length = len,
> > +.addr = buf,
> > +.flags = KVM_ARM_TAGS_TO_GUEST,
> > +};
> > +
> > +return kvm_vm_ioctl(kvm_state, KVM_ARM_MTE_COPY_TAGS, );
> > +}
> > diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
> > index 34f8daa377..bbb833d6c6 100644
> > --- a/target/arm/kvm_arm.h
> > +++ b/target/arm/kvm_arm.h
> > @@ -360,6 +360,8 @@ int kvm_arm_vgic_probe(void);
> >
> > void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
> > void kvm_arm_pmu_init(CPUState *cs);
> > +int kvm_arm_mte_get_tags(uint64_t ipa, uint64_t len, uint8_t *buf);
> > +int kvm_arm_mte_set_tags(uint64_t ipa, uint64_t len, uint8_t *buf);
> >
> > /**
> > * kvm_arm_pvtime_init:
>
[PATCH RESEND v2 4/6] hw/intc/arm_gicv3: Enable support for setting vGIC maintenance IRQ
Using the new VGIC KVM device attribute to set the maintenance IRQ.
This is fixed to use IRQ 25(PPI 9), as a platform decision matching
the arm64 SBSA recommendation.
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 5 +
hw/intc/arm_gicv3_common.c | 1 +
hw/intc/arm_gicv3_kvm.c| 16
include/hw/intc/arm_gicv3_common.h | 1 +
4 files changed, 23 insertions(+)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index aa2bbd14e0..92d46ebcfe 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -663,6 +663,11 @@ static void create_gic(VirtMachineState *vms)
qdev_prop_set_uint32(vms->gic, "redist-region-count[1]",
MIN(smp_cpus - redist0_count, redist1_capacity));
}
+
+if (kvm_irqchip_in_kernel()) {
+bool el2 = object_property_get_bool(OBJECT(first_cpu), "el2",
NULL);
+qdev_prop_set_bit(vms->gic, "has-virtualization-extensions", el2);
+}
} else {
if (!kvm_irqchip_in_kernel()) {
qdev_prop_set_bit(vms->gic, "has-virtualization-extensions",
diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c
index 58ef65f589..3ac10c8e61 100644
--- a/hw/intc/arm_gicv3_common.c
+++ b/hw/intc/arm_gicv3_common.c
@@ -495,6 +495,7 @@ static Property arm_gicv3_common_properties[] = {
DEFINE_PROP_UINT32("num-irq", GICv3State, num_irq, 32),
DEFINE_PROP_UINT32("revision", GICv3State, revision, 3),
DEFINE_PROP_BOOL("has-security-extensions", GICv3State, security_extn, 0),
+DEFINE_PROP_BOOL("has-virtualization-extensions", GICv3State, virt_extn,
0),
DEFINE_PROP_ARRAY("redist-region-count", GICv3State, nb_redist_regions,
redist_region_count, qdev_prop_uint32, uint32_t),
DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
index 65a4c880a3..1e1ca66e2c 100644
--- a/hw/intc/arm_gicv3_kvm.c
+++ b/hw/intc/arm_gicv3_kvm.c
@@ -826,6 +826,22 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error
**errp)
kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true, _abort);
+if (s->virt_extn) {
+bool maint_irq_allowed;
+uint32_t maint_irq = 25;
+
+maint_irq_allowed =
+kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ,
0);
+if (!maint_irq_allowed) {
+error_setg(errp, "VGICv3 setting maintenance IRQ are not "
+ "supported by this host kernel");
+return;
+}
+
+kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ,
+ 0, _irq, true, _abort);
+}
+
kvm_arm_register_device(>iomem_dist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
KVM_VGIC_V3_ADDR_TYPE_DIST, s->dev_fd, 0);
diff --git a/include/hw/intc/arm_gicv3_common.h
b/include/hw/intc/arm_gicv3_common.h
index 91491a2f66..921ddc2c5f 100644
--- a/include/hw/intc/arm_gicv3_common.h
+++ b/include/hw/intc/arm_gicv3_common.h
@@ -220,6 +220,7 @@ struct GICv3State {
uint32_t num_irq;
uint32_t revision;
bool security_extn;
+bool virt_extn;
bool irq_reset_nonsecure;
bool gicd_no_migration_shift_bug;
--
2.17.1
[PATCH RESEND v2 6/6] target/arm: Add vCPU feature 'el2' test.
Signed-off-by: Haibo Xu
---
target/arm/monitor.c | 2 +-
tests/qtest/arm-cpu-features.c | 9 +
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/target/arm/monitor.c b/target/arm/monitor.c
index 80c64fa355..6c39238925 100644
--- a/target/arm/monitor.c
+++ b/target/arm/monitor.c
@@ -90,7 +90,7 @@ QEMU_BUILD_BUG_ON(ARM_MAX_VQ > 16);
* then the order that considers those dependencies must be used.
*/
static const char *cpu_model_advertised_features[] = {
-"aarch64", "pmu", "sve",
+"aarch64", "pmu", "sve", "el2",
"sve128", "sve256", "sve384", "sve512",
"sve640", "sve768", "sve896", "sve1024", "sve1152", "sve1280",
"sve1408", "sve1536", "sve1664", "sve1792", "sve1920", "sve2048",
diff --git a/tests/qtest/arm-cpu-features.c b/tests/qtest/arm-cpu-features.c
index 8252b85bb8..be07bf0c76 100644
--- a/tests/qtest/arm-cpu-features.c
+++ b/tests/qtest/arm-cpu-features.c
@@ -509,6 +509,7 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
if (g_str_equal(qtest_get_arch(), "aarch64")) {
bool kvm_supports_steal_time;
bool kvm_supports_sve;
+bool kvm_supports_el2;
char max_name[8], name[8];
uint32_t max_vq, vq;
uint64_t vls;
@@ -533,10 +534,12 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
*/
assert_has_feature(qts, "host", "kvm-steal-time");
assert_has_feature(qts, "host", "sve");
+assert_has_feature(qts, "host", "el2");
resp = do_query_no_props(qts, "host");
kvm_supports_steal_time = resp_get_feature(resp, "kvm-steal-time");
kvm_supports_sve = resp_get_feature(resp, "sve");
+kvm_supports_el2 = resp_get_feature(resp, "el2");
vls = resp_get_sve_vls(resp);
qobject_unref(resp);
@@ -602,11 +605,17 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
} else {
g_assert(vls == 0);
}
+
+if (kvm_supports_el2) {
+assert_set_feature(qts, "host", "el2", false);
+assert_set_feature(qts, "host", "el2", true);
+}
} else {
assert_has_not_feature(qts, "host", "aarch64");
assert_has_not_feature(qts, "host", "pmu");
assert_has_not_feature(qts, "host", "sve");
assert_has_not_feature(qts, "host", "kvm-steal-time");
+assert_has_not_feature(qts, "host", "el2");
}
qtest_quit(qts);
--
2.17.1
[PATCH RESEND v2 3/6] target/arm/kvm: Add an option to turn on/off el2 support
Adds an el2=[on/off] option to enable/disable el2(nested virtualization)
support in KVM guest vCPU.
Signed-off-by: Haibo Xu
---
target/arm/cpu.c | 11 ++
target/arm/cpu.h | 4
target/arm/cpu64.c | 52 ++
3 files changed, 67 insertions(+)
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index ae04884408..30cc330f50 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1349,6 +1349,17 @@ void arm_cpu_finalize_features(ARMCPU *cpu, Error **errp)
return;
}
}
+
+/*
+ * Currently, vCPU feature 'el2' only supported in KVM mode.
+ */
+if (kvm_enabled()) {
+arm_cpu_el2_finalize(cpu, _err);
+if (local_err != NULL) {
+error_propagate(errp, local_err);
+return;
+}
+}
}
if (kvm_enabled()) {
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 193a49ec7f..19fa9cfbfd 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -203,10 +203,12 @@ typedef struct {
# define ARM_MAX_VQ16
void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp);
void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp);
+void arm_cpu_el2_finalize(ARMCPU *cpu, Error **errp);
#else
# define ARM_MAX_VQ1
static inline void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) { }
static inline void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp) { }
+static inline void arm_cpu_el2_finalize(ARMCPU *cpu, Error **errp) { }
#endif
typedef struct ARMVectorReg {
@@ -1058,6 +1060,7 @@ void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq);
void aarch64_sve_change_el(CPUARMState *env, int old_el,
int new_el, bool el0_a64);
void aarch64_add_sve_properties(Object *obj);
+void aarch64_add_el2_properties(Object *obj);
/*
* SVE registers are encoded in KVM's memory in an endianness-invariant format.
@@ -1089,6 +1092,7 @@ static inline void aarch64_sve_change_el(CPUARMState
*env, int o,
int n, bool a)
{ }
static inline void aarch64_add_sve_properties(Object *obj) { }
+static inline void aarch64_add_el2_properties(Object *obj) { }
#endif
void aarch64_sync_32_to_64(CPUARMState *env);
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index f0a9e968c9..3f3f2c5495 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -603,6 +603,58 @@ static Property arm_cpu_pauth_property =
static Property arm_cpu_pauth_impdef_property =
DEFINE_PROP_BOOL("pauth-impdef", ARMCPU, prop_pauth_impdef, false);
+void arm_cpu_el2_finalize(ARMCPU *cpu, Error **errp)
+{
+if (cpu->has_el2) {
+if (!kvm_enabled() || !kvm_arm_el2_supported()) {
+error_setg(errp, "'el2' cannot be enabled on this host");
+return;
+}
+}
+
+if (cpu->has_el2) {
+set_feature(>env, ARM_FEATURE_EL2);
+} else {
+unset_feature(>env, ARM_FEATURE_EL2);
+}
+}
+
+static bool arm_get_el2(Object *obj, Error **errp)
+{
+ARMCPU *cpu = ARM_CPU(obj);
+
+return cpu->has_el2;
+}
+
+static void arm_set_el2(Object *obj, bool value, Error **errp)
+{
+ARMCPU *cpu = ARM_CPU(obj);
+
+if (value) {
+if (!kvm_enabled() || !kvm_arm_el2_supported()) {
+error_setg(errp, "'el2' cannot be enabled on this host");
+return;
+}
+set_feature(>env, ARM_FEATURE_EL2);
+} else {
+unset_feature(>env, ARM_FEATURE_EL2);
+}
+
+cpu->has_el2 = value;
+}
+
+void aarch64_add_el2_properties(Object *obj)
+{
+/*
+ * vCPU feature 'el2' is only available in KVM mode, and is
+ * disabled by default to keep in line with that in TCG mode.
+ */
+ARM_CPU(obj)->has_el2 = false;
+object_property_add_bool(obj, "el2", arm_get_el2, arm_set_el2);
+object_property_set_description(obj, "el2", "Set off to disable "
+"nested virtulization.");
+}
+
/* -cpu max: if KVM is enabled, like -cpu host (best possible with this host);
* otherwise, a CPU with as many features enabled as our emulation supports.
* The version of '-cpu max' for qemu-system-arm is defined in cpu.c;
--
2.17.1
[PATCH RESEND v2 5/6] target/arm/cpu: Enable 'el2' to work with host/max cpu
Turn off the 'el2' cpu property by default to keep in line with
that in TCG mode, i.e. we can now use '-cpu max|host,el2=on' to
enable the nested virtualization.
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 14 ++
target/arm/cpu.c | 3 ++-
target/arm/cpu64.c | 1 +
target/arm/kvm64.c | 10 ++
4 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 92d46ebcfe..74340e21bd 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -454,6 +454,7 @@ static void fdt_add_gic_node(VirtMachineState *vms)
{
MachineState *ms = MACHINE(vms);
char *nodename;
+bool has_el2 = object_property_get_bool(OBJECT(first_cpu), "el2", NULL);
vms->gic_phandle = qemu_fdt_alloc_phandle(ms->fdt);
qemu_fdt_setprop_cell(ms->fdt, "/", "interrupt-parent", vms->gic_phandle);
@@ -491,7 +492,7 @@ static void fdt_add_gic_node(VirtMachineState *vms)
2, vms->memmap[VIRT_HIGH_GIC_REDIST2].size);
}
-if (vms->virt) {
+if (vms->virt || has_el2) {
qemu_fdt_setprop_cells(ms->fdt, nodename, "interrupts",
GIC_FDT_IRQ_TYPE_PPI, ARCH_GIC_MAINT_IRQ,
GIC_FDT_IRQ_FLAGS_LEVEL_HI);
@@ -1911,8 +1912,8 @@ static void machvirt_init(MachineState *machine)
}
if (vms->virt && kvm_enabled()) {
-error_report("mach-virt: KVM does not support providing "
- "Virtualization extensions to the guest CPU");
+error_report("mach-virt: VM 'virtualization' feature is not supported "
+ "in KVM mode, please use CPU feature 'el2' instead");
exit(1);
}
@@ -1950,11 +1951,16 @@ static void machvirt_init(MachineState *machine)
object_property_set_bool(cpuobj, "has_el3", false, NULL);
}
-if (!vms->virt && object_property_find(cpuobj, "has_el2")) {
+if (!vms->virt && !kvm_enabled() &&
+object_property_find(cpuobj, "has_el2")) {
object_property_set_bool(cpuobj, "has_el2", false, NULL);
}
if (vms->psci_conduit != QEMU_PSCI_CONDUIT_DISABLED) {
+if (kvm_enabled() && ARM_CPU(cpuobj)->has_el2) {
+vms->psci_conduit = QEMU_PSCI_CONDUIT_SMC;
+}
+
object_property_set_int(cpuobj, "psci-conduit", vms->psci_conduit,
NULL);
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 30cc330f50..9530a2c4bf 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1099,7 +1099,7 @@ static Property arm_cpu_rvbar_property =
#ifndef CONFIG_USER_ONLY
static Property arm_cpu_has_el2_property =
-DEFINE_PROP_BOOL("has_el2", ARMCPU, has_el2, true);
+DEFINE_PROP_BOOL("has_el2", ARMCPU, has_el2, false);
static Property arm_cpu_has_el3_property =
DEFINE_PROP_BOOL("has_el3", ARMCPU, has_el3, true);
@@ -2018,6 +2018,7 @@ static void arm_host_initfn(Object *obj)
kvm_arm_set_cpu_features_from_host(cpu);
if (arm_feature(>env, ARM_FEATURE_AARCH64)) {
aarch64_add_sve_properties(obj);
+aarch64_add_el2_properties(obj);
}
arm_cpu_post_init(obj);
}
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 3f3f2c5495..ae8811d09e 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -666,6 +666,7 @@ static void aarch64_max_initfn(Object *obj)
if (kvm_enabled()) {
kvm_arm_set_cpu_features_from_host(cpu);
+aarch64_add_el2_properties(obj);
} else {
uint64_t t;
uint32_t u;
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 9cacaf2eb8..7bf892404f 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -500,6 +500,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
*/
int fdarray[3];
bool sve_supported;
+bool el2_supported;
uint64_t features = 0;
uint64_t t;
int err;
@@ -646,6 +647,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
}
sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) >
0;
+el2_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_EL2) >
0;
kvm_arm_destroy_scratch_host_vcpu(fdarray);
@@ -660,6 +662,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
*ahcf)
ahcf->isar.id_aa64pfr0 = t;
}
+/* Use the ARM_FEATURE_EL2 bit to keep inline with that in TCG mode. */
+if (el2_supported) {
+features |= 1ULL << ARM_FEATURE_EL2;
+}
+
/*
* We can assume any KVM supporting CPU is at least a v8
* with VFPv4+Neon; this in turn implies most of
[PATCH RESEND v2 2/6] target/arm/kvm: Add helper to detect el2 when using KVM
Signed-off-by: Haibo Xu
---
target/arm/kvm64.c | 5 +
target/arm/kvm_arm.h | 13 +
2 files changed, 18 insertions(+)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index dff85f6db9..9cacaf2eb8 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -721,6 +721,11 @@ bool kvm_arm_steal_time_supported(void)
return kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME);
}
+bool kvm_arm_el2_supported(void)
+{
+return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL2);
+}
+
QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1);
void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map)
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index 34f8daa377..7d7fc7981b 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -285,6 +285,14 @@ void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error
**errp);
*/
bool kvm_arm_steal_time_supported(void);
+/**
+ * kvm_arm_el2_supported:
+ *
+ * Returns: true if KVM can enable el2(nested virtualization)
+ * and false otherwise.
+ */
+bool kvm_arm_el2_supported(void);
+
/**
* kvm_arm_aarch32_supported:
*
@@ -398,6 +406,11 @@ static inline bool kvm_arm_steal_time_supported(void)
return false;
}
+static inline bool kvm_arm_el2_supported(void)
+{
+return false;
+}
+
/*
* These functions should never actually be called without KVM support.
*/
--
2.17.1
[PATCH RESEND v2 0/6] target/arm: Add nested virtualization support
v2: - Move the NV to a CPU feature flag(Andrea) - Add CPU feature 'el2' test(Andrew) Many thanks to Andrea and Andrew for their comments! This series add support for ARMv8.3/8.4 nested virtualization support in KVM mode. It's based on Marc Zyngier's kernel KVM patches[1], and has been tested on a FVP model to run a L2 guest with Qemu. Now the feature can be enabled by "-M virt,accel=kvm -cpu host,el2=on" when starting a VM. [1] https://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git/log/?h=kvm-arm64/nv-5.12-WIP Haibo Xu (6): Update linux header with new arm64 NV macro target/arm/kvm: Add helper to detect el2 when using KVM target/arm/kvm: Add an option to turn on/off el2 support hw/intc/arm_gicv3: Enable support for setting vGIC maintenance IRQ target/arm/cpu: Enable 'el2' to work with host/max cpu target/arm: Add vCPU feature 'el2' test. hw/arm/virt.c | 19 --- hw/intc/arm_gicv3_common.c | 1 + hw/intc/arm_gicv3_kvm.c| 16 + include/hw/intc/arm_gicv3_common.h | 1 + linux-headers/asm-arm64/kvm.h | 2 ++ linux-headers/linux/kvm.h | 1 + target/arm/cpu.c | 14 +++- target/arm/cpu.h | 4 +++ target/arm/cpu64.c | 53 ++ target/arm/kvm64.c | 15 + target/arm/kvm_arm.h | 13 target/arm/monitor.c | 2 +- tests/qtest/arm-cpu-features.c | 9 + 13 files changed, 144 insertions(+), 6 deletions(-) -- 2.17.1
[PATCH RESEND v2 1/6] Update linux header with new arm64 NV macro
Signed-off-by: Haibo Xu
---
linux-headers/asm-arm64/kvm.h | 2 ++
linux-headers/linux/kvm.h | 1 +
2 files changed, 3 insertions(+)
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index b6a0eaa32a..77b995a26c 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -106,6 +106,7 @@ struct kvm_regs {
#define KVM_ARM_VCPU_SVE 4 /* enable SVE for this CPU */
#define KVM_ARM_VCPU_PTRAUTH_ADDRESS 5 /* VCPU uses address authentication */
#define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */
+#define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */
struct kvm_vcpu_init {
__u32 target;
@@ -334,6 +335,7 @@ struct kvm_vcpu_events {
#define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6
#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7
#define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8
+#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9
#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10
#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \
(0x3fULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 020b62a619..ce4630c4db 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1056,6 +1056,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
#define KVM_CAP_SYS_HYPERV_CPUID 191
#define KVM_CAP_DIRTY_LOG_RING 192
+#define KVM_CAP_ARM_EL2 193
#ifdef KVM_CAP_IRQ_ROUTING
--
2.17.1
[PATCH v2 3/6] target/arm/kvm: Add an option to turn on/off el2 support
Adds an el2=[on/off] option to enable/disable el2(nested virtualization)
support in KVM guest vCPU.
Signed-off-by: Haibo Xu
---
target/arm/cpu.c | 11 ++
target/arm/cpu.h | 4
target/arm/cpu64.c | 52 ++
3 files changed, 67 insertions(+)
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index ae04884408..30cc330f50 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1349,6 +1349,17 @@ void arm_cpu_finalize_features(ARMCPU *cpu, Error **errp)
return;
}
}
+
+/*
+ * Currently, vCPU feature 'el2' only supported in KVM mode.
+ */
+if (kvm_enabled()) {
+arm_cpu_el2_finalize(cpu, _err);
+if (local_err != NULL) {
+error_propagate(errp, local_err);
+return;
+}
+}
}
if (kvm_enabled()) {
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 193a49ec7f..19fa9cfbfd 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -203,10 +203,12 @@ typedef struct {
# define ARM_MAX_VQ16
void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp);
void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp);
+void arm_cpu_el2_finalize(ARMCPU *cpu, Error **errp);
#else
# define ARM_MAX_VQ1
static inline void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) { }
static inline void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp) { }
+static inline void arm_cpu_el2_finalize(ARMCPU *cpu, Error **errp) { }
#endif
typedef struct ARMVectorReg {
@@ -1058,6 +1060,7 @@ void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq);
void aarch64_sve_change_el(CPUARMState *env, int old_el,
int new_el, bool el0_a64);
void aarch64_add_sve_properties(Object *obj);
+void aarch64_add_el2_properties(Object *obj);
/*
* SVE registers are encoded in KVM's memory in an endianness-invariant format.
@@ -1089,6 +1092,7 @@ static inline void aarch64_sve_change_el(CPUARMState
*env, int o,
int n, bool a)
{ }
static inline void aarch64_add_sve_properties(Object *obj) { }
+static inline void aarch64_add_el2_properties(Object *obj) { }
#endif
void aarch64_sync_32_to_64(CPUARMState *env);
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index f0a9e968c9..3f3f2c5495 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -603,6 +603,58 @@ static Property arm_cpu_pauth_property =
static Property arm_cpu_pauth_impdef_property =
DEFINE_PROP_BOOL("pauth-impdef", ARMCPU, prop_pauth_impdef, false);
+void arm_cpu_el2_finalize(ARMCPU *cpu, Error **errp)
+{
+if (cpu->has_el2) {
+if (!kvm_enabled() || !kvm_arm_el2_supported()) {
+error_setg(errp, "'el2' cannot be enabled on this host");
+return;
+}
+}
+
+if (cpu->has_el2) {
+set_feature(>env, ARM_FEATURE_EL2);
+} else {
+unset_feature(>env, ARM_FEATURE_EL2);
+}
+}
+
+static bool arm_get_el2(Object *obj, Error **errp)
+{
+ARMCPU *cpu = ARM_CPU(obj);
+
+return cpu->has_el2;
+}
+
+static void arm_set_el2(Object *obj, bool value, Error **errp)
+{
+ARMCPU *cpu = ARM_CPU(obj);
+
+if (value) {
+if (!kvm_enabled() || !kvm_arm_el2_supported()) {
+error_setg(errp, "'el2' cannot be enabled on this host");
+return;
+}
+set_feature(>env, ARM_FEATURE_EL2);
+} else {
+unset_feature(>env, ARM_FEATURE_EL2);
+}
+
+cpu->has_el2 = value;
+}
+
+void aarch64_add_el2_properties(Object *obj)
+{
+/*
+ * vCPU feature 'el2' is only available in KVM mode, and is
+ * disabled by default to keep in line with that in TCG mode.
+ */
+ARM_CPU(obj)->has_el2 = false;
+object_property_add_bool(obj, "el2", arm_get_el2, arm_set_el2);
+object_property_set_description(obj, "el2", "Set off to disable "
+"nested virtulization.");
+}
+
/* -cpu max: if KVM is enabled, like -cpu host (best possible with this host);
* otherwise, a CPU with as many features enabled as our emulation supports.
* The version of '-cpu max' for qemu-system-arm is defined in cpu.c;
--
2.17.1
[PATCH v2 6/6] target/arm: Add vCPU feature 'el2' test.
Signed-off-by: Haibo Xu
---
target/arm/monitor.c | 2 +-
tests/qtest/arm-cpu-features.c | 9 +
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/target/arm/monitor.c b/target/arm/monitor.c
index 80c64fa355..6c39238925 100644
--- a/target/arm/monitor.c
+++ b/target/arm/monitor.c
@@ -90,7 +90,7 @@ QEMU_BUILD_BUG_ON(ARM_MAX_VQ > 16);
* then the order that considers those dependencies must be used.
*/
static const char *cpu_model_advertised_features[] = {
-"aarch64", "pmu", "sve",
+"aarch64", "pmu", "sve", "el2",
"sve128", "sve256", "sve384", "sve512",
"sve640", "sve768", "sve896", "sve1024", "sve1152", "sve1280",
"sve1408", "sve1536", "sve1664", "sve1792", "sve1920", "sve2048",
diff --git a/tests/qtest/arm-cpu-features.c b/tests/qtest/arm-cpu-features.c
index 8252b85bb8..be07bf0c76 100644
--- a/tests/qtest/arm-cpu-features.c
+++ b/tests/qtest/arm-cpu-features.c
@@ -509,6 +509,7 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
if (g_str_equal(qtest_get_arch(), "aarch64")) {
bool kvm_supports_steal_time;
bool kvm_supports_sve;
+bool kvm_supports_el2;
char max_name[8], name[8];
uint32_t max_vq, vq;
uint64_t vls;
@@ -533,10 +534,12 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
*/
assert_has_feature(qts, "host", "kvm-steal-time");
assert_has_feature(qts, "host", "sve");
+assert_has_feature(qts, "host", "el2");
resp = do_query_no_props(qts, "host");
kvm_supports_steal_time = resp_get_feature(resp, "kvm-steal-time");
kvm_supports_sve = resp_get_feature(resp, "sve");
+kvm_supports_el2 = resp_get_feature(resp, "el2");
vls = resp_get_sve_vls(resp);
qobject_unref(resp);
@@ -602,11 +605,17 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
} else {
g_assert(vls == 0);
}
+
+if (kvm_supports_el2) {
+assert_set_feature(qts, "host", "el2", false);
+assert_set_feature(qts, "host", "el2", true);
+}
} else {
assert_has_not_feature(qts, "host", "aarch64");
assert_has_not_feature(qts, "host", "pmu");
assert_has_not_feature(qts, "host", "sve");
assert_has_not_feature(qts, "host", "kvm-steal-time");
+assert_has_not_feature(qts, "host", "el2");
}
qtest_quit(qts);
--
2.17.1
[PATCH v2 5/6] target/arm/cpu: Enable 'el2' to work with host/max cpu
Turn off the 'el2' cpu property by default to keep in line with
that in TCG mode, i.e. we can now use '-cpu max|host,el2=on' to
enable the nested virtualization.
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 14 ++
target/arm/cpu.c | 3 ++-
target/arm/cpu64.c | 1 +
target/arm/kvm64.c | 10 ++
4 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 92d46ebcfe..74340e21bd 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -454,6 +454,7 @@ static void fdt_add_gic_node(VirtMachineState *vms)
{
MachineState *ms = MACHINE(vms);
char *nodename;
+bool has_el2 = object_property_get_bool(OBJECT(first_cpu), "el2", NULL);
vms->gic_phandle = qemu_fdt_alloc_phandle(ms->fdt);
qemu_fdt_setprop_cell(ms->fdt, "/", "interrupt-parent", vms->gic_phandle);
@@ -491,7 +492,7 @@ static void fdt_add_gic_node(VirtMachineState *vms)
2, vms->memmap[VIRT_HIGH_GIC_REDIST2].size);
}
-if (vms->virt) {
+if (vms->virt || has_el2) {
qemu_fdt_setprop_cells(ms->fdt, nodename, "interrupts",
GIC_FDT_IRQ_TYPE_PPI, ARCH_GIC_MAINT_IRQ,
GIC_FDT_IRQ_FLAGS_LEVEL_HI);
@@ -1911,8 +1912,8 @@ static void machvirt_init(MachineState *machine)
}
if (vms->virt && kvm_enabled()) {
-error_report("mach-virt: KVM does not support providing "
- "Virtualization extensions to the guest CPU");
+error_report("mach-virt: VM 'virtualization' feature is not supported "
+ "in KVM mode, please use CPU feature 'el2' instead");
exit(1);
}
@@ -1950,11 +1951,16 @@ static void machvirt_init(MachineState *machine)
object_property_set_bool(cpuobj, "has_el3", false, NULL);
}
-if (!vms->virt && object_property_find(cpuobj, "has_el2")) {
+if (!vms->virt && !kvm_enabled() &&
+object_property_find(cpuobj, "has_el2")) {
object_property_set_bool(cpuobj, "has_el2", false, NULL);
}
if (vms->psci_conduit != QEMU_PSCI_CONDUIT_DISABLED) {
+if (kvm_enabled() && ARM_CPU(cpuobj)->has_el2) {
+vms->psci_conduit = QEMU_PSCI_CONDUIT_SMC;
+}
+
object_property_set_int(cpuobj, "psci-conduit", vms->psci_conduit,
NULL);
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 30cc330f50..9530a2c4bf 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1099,7 +1099,7 @@ static Property arm_cpu_rvbar_property =
#ifndef CONFIG_USER_ONLY
static Property arm_cpu_has_el2_property =
-DEFINE_PROP_BOOL("has_el2", ARMCPU, has_el2, true);
+DEFINE_PROP_BOOL("has_el2", ARMCPU, has_el2, false);
static Property arm_cpu_has_el3_property =
DEFINE_PROP_BOOL("has_el3", ARMCPU, has_el3, true);
@@ -2018,6 +2018,7 @@ static void arm_host_initfn(Object *obj)
kvm_arm_set_cpu_features_from_host(cpu);
if (arm_feature(>env, ARM_FEATURE_AARCH64)) {
aarch64_add_sve_properties(obj);
+aarch64_add_el2_properties(obj);
}
arm_cpu_post_init(obj);
}
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 3f3f2c5495..ae8811d09e 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -666,6 +666,7 @@ static void aarch64_max_initfn(Object *obj)
if (kvm_enabled()) {
kvm_arm_set_cpu_features_from_host(cpu);
+aarch64_add_el2_properties(obj);
} else {
uint64_t t;
uint32_t u;
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 9cacaf2eb8..7bf892404f 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -500,6 +500,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
*/
int fdarray[3];
bool sve_supported;
+bool el2_supported;
uint64_t features = 0;
uint64_t t;
int err;
@@ -646,6 +647,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
}
sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) >
0;
+el2_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_EL2) >
0;
kvm_arm_destroy_scratch_host_vcpu(fdarray);
@@ -660,6 +662,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
*ahcf)
ahcf->isar.id_aa64pfr0 = t;
}
+/* Use the ARM_FEATURE_EL2 bit to keep inline with that in TCG mode. */
+if (el2_supported) {
+features |= 1ULL << ARM_FEATURE_EL2;
+}
+
/*
* We can assume any KVM supporting CPU is at least a v8
* with VFPv4+Neon; this in turn implies most of
[PATCH v2 2/6] target/arm/kvm: Add helper to detect el2 when using KVM
Signed-off-by: Haibo Xu
---
target/arm/kvm64.c | 5 +
target/arm/kvm_arm.h | 13 +
2 files changed, 18 insertions(+)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index dff85f6db9..9cacaf2eb8 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -721,6 +721,11 @@ bool kvm_arm_steal_time_supported(void)
return kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME);
}
+bool kvm_arm_el2_supported(void)
+{
+return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL2);
+}
+
QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1);
void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map)
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index 34f8daa377..7d7fc7981b 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -285,6 +285,14 @@ void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error
**errp);
*/
bool kvm_arm_steal_time_supported(void);
+/**
+ * kvm_arm_el2_supported:
+ *
+ * Returns: true if KVM can enable el2(nested virtualization)
+ * and false otherwise.
+ */
+bool kvm_arm_el2_supported(void);
+
/**
* kvm_arm_aarch32_supported:
*
@@ -398,6 +406,11 @@ static inline bool kvm_arm_steal_time_supported(void)
return false;
}
+static inline bool kvm_arm_el2_supported(void)
+{
+return false;
+}
+
/*
* These functions should never actually be called without KVM support.
*/
--
2.17.1
[PATCH v2 4/6] hw/intc/arm_gicv3: Enable support for setting vGIC maintenance IRQ
Using the new VGIC KVM device attribute to set the maintenance IRQ.
This is fixed to use IRQ 25(PPI 9), as a platform decision matching
the arm64 SBSA recommendation.
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 5 +
hw/intc/arm_gicv3_common.c | 1 +
hw/intc/arm_gicv3_kvm.c| 16
include/hw/intc/arm_gicv3_common.h | 1 +
4 files changed, 23 insertions(+)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index aa2bbd14e0..92d46ebcfe 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -663,6 +663,11 @@ static void create_gic(VirtMachineState *vms)
qdev_prop_set_uint32(vms->gic, "redist-region-count[1]",
MIN(smp_cpus - redist0_count, redist1_capacity));
}
+
+if (kvm_irqchip_in_kernel()) {
+bool el2 = object_property_get_bool(OBJECT(first_cpu), "el2",
NULL);
+qdev_prop_set_bit(vms->gic, "has-virtualization-extensions", el2);
+}
} else {
if (!kvm_irqchip_in_kernel()) {
qdev_prop_set_bit(vms->gic, "has-virtualization-extensions",
diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c
index 58ef65f589..3ac10c8e61 100644
--- a/hw/intc/arm_gicv3_common.c
+++ b/hw/intc/arm_gicv3_common.c
@@ -495,6 +495,7 @@ static Property arm_gicv3_common_properties[] = {
DEFINE_PROP_UINT32("num-irq", GICv3State, num_irq, 32),
DEFINE_PROP_UINT32("revision", GICv3State, revision, 3),
DEFINE_PROP_BOOL("has-security-extensions", GICv3State, security_extn, 0),
+DEFINE_PROP_BOOL("has-virtualization-extensions", GICv3State, virt_extn,
0),
DEFINE_PROP_ARRAY("redist-region-count", GICv3State, nb_redist_regions,
redist_region_count, qdev_prop_uint32, uint32_t),
DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
index 65a4c880a3..1e1ca66e2c 100644
--- a/hw/intc/arm_gicv3_kvm.c
+++ b/hw/intc/arm_gicv3_kvm.c
@@ -826,6 +826,22 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error
**errp)
kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true, _abort);
+if (s->virt_extn) {
+bool maint_irq_allowed;
+uint32_t maint_irq = 25;
+
+maint_irq_allowed =
+kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ,
0);
+if (!maint_irq_allowed) {
+error_setg(errp, "VGICv3 setting maintenance IRQ are not "
+ "supported by this host kernel");
+return;
+}
+
+kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ,
+ 0, _irq, true, _abort);
+}
+
kvm_arm_register_device(>iomem_dist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
KVM_VGIC_V3_ADDR_TYPE_DIST, s->dev_fd, 0);
diff --git a/include/hw/intc/arm_gicv3_common.h
b/include/hw/intc/arm_gicv3_common.h
index 91491a2f66..921ddc2c5f 100644
--- a/include/hw/intc/arm_gicv3_common.h
+++ b/include/hw/intc/arm_gicv3_common.h
@@ -220,6 +220,7 @@ struct GICv3State {
uint32_t num_irq;
uint32_t revision;
bool security_extn;
+bool virt_extn;
bool irq_reset_nonsecure;
bool gicd_no_migration_shift_bug;
--
2.17.1
[PATCH v2 0/6] target/arm: Add nested virtualization support
v2: - Move the NV to a CPU feature flag(Andrea) - Add CPU feature 'el2' test(Andrew) Many thanks to Andrea and Andrew for their comments! This series add support for ARMv8.3/8.4 nested virtualization support in KVM mode. It's based on Marc Zyngier's kernel KVM patches[1], and has been tested on a FVP model to run a L2 guest with Qemu. Now the feature can be enabled by "-M virt,accel=kvm,virtualization=on" when starting a VM. [1] https://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git/log/?h=kvm-arm64/nv-5.12-WIP Haibo Xu (6): Update linux header with new arm64 NV macro target/arm/kvm: Add helper to detect el2 when using KVM target/arm/kvm: Add an option to turn on/off el2 support hw/intc/arm_gicv3: Enable support for setting vGIC maintenance IRQ target/arm/cpu: Enable 'el2' to work with host/max cpu target/arm: Add vCPU feature 'el2' test. hw/arm/virt.c | 19 --- hw/intc/arm_gicv3_common.c | 1 + hw/intc/arm_gicv3_kvm.c| 16 + include/hw/intc/arm_gicv3_common.h | 1 + linux-headers/asm-arm64/kvm.h | 2 ++ linux-headers/linux/kvm.h | 1 + target/arm/cpu.c | 14 +++- target/arm/cpu.h | 4 +++ target/arm/cpu64.c | 53 ++ target/arm/kvm64.c | 15 + target/arm/kvm_arm.h | 13 target/arm/monitor.c | 2 +- tests/qtest/arm-cpu-features.c | 9 + 13 files changed, 144 insertions(+), 6 deletions(-) -- 2.17.1
[PATCH v2 1/6] Update linux header with new arm64 NV macro
Signed-off-by: Haibo Xu
---
linux-headers/asm-arm64/kvm.h | 2 ++
linux-headers/linux/kvm.h | 1 +
2 files changed, 3 insertions(+)
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index b6a0eaa32a..77b995a26c 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -106,6 +106,7 @@ struct kvm_regs {
#define KVM_ARM_VCPU_SVE 4 /* enable SVE for this CPU */
#define KVM_ARM_VCPU_PTRAUTH_ADDRESS 5 /* VCPU uses address authentication */
#define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */
+#define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */
struct kvm_vcpu_init {
__u32 target;
@@ -334,6 +335,7 @@ struct kvm_vcpu_events {
#define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6
#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7
#define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8
+#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9
#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10
#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \
(0x3fULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 020b62a619..ce4630c4db 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1056,6 +1056,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
#define KVM_CAP_SYS_HYPERV_CPUID 191
#define KVM_CAP_DIRTY_LOG_RING 192
+#define KVM_CAP_ARM_EL2 193
#ifdef KVM_CAP_IRQ_ROUTING
--
2.17.1
Re: [PATCH 3/3] Enable nested virtualization support in arm64 KVM mode
On Mon, 22 Mar 2021 at 18:49, Andrew Jones wrote:
>
> On Mon, Mar 22, 2021 at 10:07:26AM +, Haibo Xu wrote:
> > Add support for arm64 el2 in qemu KVM mode(nested virtualization).
> > This feature is disabled by default, just as that in TCG mode, and
> > can be enabled by "-M virt,accel=kvm,virtualization=on" when starting
> > a VM.
> >
> > Signed-off-by: Haibo Xu
> > ---
> > hw/arm/virt.c| 11 ---
> > target/arm/cpu.h | 8
> > target/arm/kvm64.c | 14 ++
> > target/arm/kvm_arm.h | 28
> > 4 files changed, 58 insertions(+), 3 deletions(-)
> >
> > diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> > index aa2bbd14e0..72e60348d5 100644
> > --- a/hw/arm/virt.c
> > +++ b/hw/arm/virt.c
> > @@ -663,6 +663,11 @@ static void create_gic(VirtMachineState *vms)
> > qdev_prop_set_uint32(vms->gic, "redist-region-count[1]",
> > MIN(smp_cpus - redist0_count, redist1_capacity));
> > }
> > +
> > +if (kvm_irqchip_in_kernel()) {
> > +qdev_prop_set_bit(vms->gic, "has-virtualization-extensions",
> > + vms->virt);
> > +}
> > } else {
> > if (!kvm_irqchip_in_kernel()) {
> > qdev_prop_set_bit(vms->gic, "has-virtualization-extensions",
> > @@ -1905,9 +1910,9 @@ static void machvirt_init(MachineState *machine)
> > exit(1);
> > }
> >
> > -if (vms->virt && kvm_enabled()) {
> > -error_report("mach-virt: KVM does not support providing "
> > - "Virtualization extensions to the guest CPU");
> > +if (vms->virt && kvm_enabled() && !kvm_arm_nested_virt_supported()) {
> > +error_report("mach-virt: nested virtualization requested, "
> > + "but not supported by the host.");
> > exit(1);
> > }
> >
> > diff --git a/target/arm/cpu.h b/target/arm/cpu.h
> > index 193a49ec7f..377187152b 100644
> > --- a/target/arm/cpu.h
> > +++ b/target/arm/cpu.h
> > @@ -4182,6 +4182,14 @@ static inline bool isar_feature_aa64_ssbs(const
> > ARMISARegisters *id)
> > return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, SSBS) != 0;
> > }
> >
> > +/*
> > + * Currently we don't differentiate between the ARMv8.3-NV and ARMv8.4-NV.
> > + */
> > +static inline bool isar_feature_aa64_nv(const ARMISARegisters *id)
> > +{
> > +return FIELD_EX64(id->id_aa64mmfr2, ID_AA64MMFR2, NV) != 0;
> > +}
>
> What calls this function?
>
Sorry for the noise here! This function should be deleted in this patch!
Previously, I wanted to follow the SVE vCPU feature when adding the NV support,
and the above function is supposed to be used for the feature probe.
Thanks,
Haibo
> > +
> > /*
> > * Feature tests for "does this exist in either 32-bit or 64-bit?"
> > */
> > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> > index dff85f6db9..2810104dea 100644
> > --- a/target/arm/kvm64.c
> > +++ b/target/arm/kvm64.c
> > @@ -500,6 +500,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> > *ahcf)
> > */
> > int fdarray[3];
> > bool sve_supported;
> > +bool el2_supported;
> > uint64_t features = 0;
> > uint64_t t;
> > int err;
> > @@ -646,6 +647,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> > *ahcf)
> > }
> >
> > sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION,
> > KVM_CAP_ARM_SVE) > 0;
> > +el2_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION,
> > KVM_CAP_ARM_EL2) > 0;
> >
> > kvm_arm_destroy_scratch_host_vcpu(fdarray);
> >
> > @@ -671,6 +673,10 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> > *ahcf)
> > features |= 1ULL << ARM_FEATURE_PMU;
> > features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
> >
> > +if (el2_supported) {
> > +features |= 1ULL << ARM_FEATURE_EL2;
> > +}
> > +
> > ahcf->features = features;
> >
> > return true;
> > @@ -721,6 +727,11 @@ bool kvm_arm_steal_time_supported(void)
> > return kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME);
> > }
> >
> > +bool kvm_arm_nested_virt_supported(void)
> > +{
> > +return kvm_check_extension
Re: [PATCH 0/3] target/arm: Add nested virtualization support
On Tue, 23 Mar 2021 at 00:32, Andrew Jones wrote: > > On Mon, Mar 22, 2021 at 04:42:23PM +0100, Andrea Bolognani wrote: > > On Mon, 2021-03-22 at 10:07 +0000, Haibo Xu wrote: > > > This series add support for ARMv8.3/8.4 nested virtualization support > > > in KVM mode. It's based on Marc Zyngier's kernel KVM patches[1], and > > > has been tested on a FVP model to run a L2 guest with Qemu. Now the > > > feature can be enabled by "-M virt,accel=kvm,virtualization=on" when > > > starting a VM. > > > > Why the need to enable this explicitly? AFAIK, that's not necessary > > for any other architecture: on x86, you just need to make sure you're > > using '-cpu host' and pass a parameter to the kernel module. > > > > Even assuming this can't be enabled transparently, wouldn't its > > availability it be controlled by a CPU feature flag, similar to what > > already happens for SVE and PMU, rather than a machine type option? > > I 100% agree. We should control this feature with a CPU feature property. > NV is a CPU feature, after all. Also, we should add it to the properties > that we can probe in cpu_model_advertised_features[]. > Thanks so much for the comments! Yes, NV should be a vCPU feature, just as the kernel macro KVM_ARM_VCPU_HAS_EL2 implies. To implement it as a VM feature here just want to reuse the codes in TCG mode which emulate a guest CPU with virtualization extension support. I will change the NV in KVM mode to a vCPU feature in the next version. @Peter Maydell and @Richard Henderson, shall we change that in TCG mode as well? Thanks, Haibo > Thanks, > drew > > > > > That would also address the discoverability issue: unless I'm > > mistaken (which I very well might be :), with the current > > implementation there's no way to tell whether nested KVM will be > > usable short of trying and seeing whether QEMU errors out. > > > > -- > > Andrea Bolognani / Red Hat / Virtualization > > > > >
[PATCH 1/3] Update linux header with new arm64 NV macro.
Signed-off-by: Haibo Xu
---
linux-headers/asm-arm64/kvm.h | 2 ++
linux-headers/linux/kvm.h | 1 +
2 files changed, 3 insertions(+)
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index b6a0eaa32a..77b995a26c 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -106,6 +106,7 @@ struct kvm_regs {
#define KVM_ARM_VCPU_SVE 4 /* enable SVE for this CPU */
#define KVM_ARM_VCPU_PTRAUTH_ADDRESS 5 /* VCPU uses address authentication */
#define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */
+#define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */
struct kvm_vcpu_init {
__u32 target;
@@ -334,6 +335,7 @@ struct kvm_vcpu_events {
#define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6
#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7
#define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8
+#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9
#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10
#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \
(0x3fULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 020b62a619..ce4630c4db 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1056,6 +1056,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
#define KVM_CAP_SYS_HYPERV_CPUID 191
#define KVM_CAP_DIRTY_LOG_RING 192
+#define KVM_CAP_ARM_EL2 193
#ifdef KVM_CAP_IRQ_ROUTING
--
2.17.1
[PATCH 3/3] Enable nested virtualization support in arm64 KVM mode
Add support for arm64 el2 in qemu KVM mode(nested virtualization).
This feature is disabled by default, just as that in TCG mode, and
can be enabled by "-M virt,accel=kvm,virtualization=on" when starting
a VM.
Signed-off-by: Haibo Xu
---
hw/arm/virt.c| 11 ---
target/arm/cpu.h | 8
target/arm/kvm64.c | 14 ++
target/arm/kvm_arm.h | 28
4 files changed, 58 insertions(+), 3 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index aa2bbd14e0..72e60348d5 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -663,6 +663,11 @@ static void create_gic(VirtMachineState *vms)
qdev_prop_set_uint32(vms->gic, "redist-region-count[1]",
MIN(smp_cpus - redist0_count, redist1_capacity));
}
+
+if (kvm_irqchip_in_kernel()) {
+qdev_prop_set_bit(vms->gic, "has-virtualization-extensions",
+ vms->virt);
+}
} else {
if (!kvm_irqchip_in_kernel()) {
qdev_prop_set_bit(vms->gic, "has-virtualization-extensions",
@@ -1905,9 +1910,9 @@ static void machvirt_init(MachineState *machine)
exit(1);
}
-if (vms->virt && kvm_enabled()) {
-error_report("mach-virt: KVM does not support providing "
- "Virtualization extensions to the guest CPU");
+if (vms->virt && kvm_enabled() && !kvm_arm_nested_virt_supported()) {
+error_report("mach-virt: nested virtualization requested, "
+ "but not supported by the host.");
exit(1);
}
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 193a49ec7f..377187152b 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -4182,6 +4182,14 @@ static inline bool isar_feature_aa64_ssbs(const
ARMISARegisters *id)
return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, SSBS) != 0;
}
+/*
+ * Currently we don't differentiate between the ARMv8.3-NV and ARMv8.4-NV.
+ */
+static inline bool isar_feature_aa64_nv(const ARMISARegisters *id)
+{
+return FIELD_EX64(id->id_aa64mmfr2, ID_AA64MMFR2, NV) != 0;
+}
+
/*
* Feature tests for "does this exist in either 32-bit or 64-bit?"
*/
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index dff85f6db9..2810104dea 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -500,6 +500,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
*/
int fdarray[3];
bool sve_supported;
+bool el2_supported;
uint64_t features = 0;
uint64_t t;
int err;
@@ -646,6 +647,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
}
sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) >
0;
+el2_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_EL2) >
0;
kvm_arm_destroy_scratch_host_vcpu(fdarray);
@@ -671,6 +673,10 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
*ahcf)
features |= 1ULL << ARM_FEATURE_PMU;
features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
+if (el2_supported) {
+features |= 1ULL << ARM_FEATURE_EL2;
+}
+
ahcf->features = features;
return true;
@@ -721,6 +727,11 @@ bool kvm_arm_steal_time_supported(void)
return kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME);
}
+bool kvm_arm_nested_virt_supported(void)
+{
+return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL2);
+}
+
QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1);
void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map)
@@ -856,6 +867,9 @@ int kvm_arch_init_vcpu(CPUState *cs)
assert(kvm_arm_sve_supported());
cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
}
+if (cpu->has_el2) {
+cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_HAS_EL2;
+}
/* Do KVM_ARM_VCPU_INIT ioctl */
ret = kvm_arm_vcpu_init(cs);
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index 34f8daa377..da3a3d5920 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -285,6 +285,24 @@ void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error
**errp);
*/
bool kvm_arm_steal_time_supported(void);
+/**
+ * kvm_arm_nested_virt_finalize:
+ * @cpu: ARMCPU for which to finalize nested-virt
+ * @errp: Pointer to Error* for error propagation
+ *
+ * Validate the nested-virt property selection and set its default
+ * based on KVM support and guest configuration.
+ */
+void kvm_arm_nested_virt_finalize(ARMCPU *cpu, Error **errp);
+
+/**
+ * kvm_arm_nested_virt_supported:
+ *
+ * Returns: true if KVM can enable nested virtualization
+ * and false otherwise.
+ */
+bool kvm_arm_nested_virt_supported(void);
+
/**
* kvm_arm_aarch32_supported:
*
@@ -398,6 +416,11 @@ static inline bool kvm_arm_steal_time_supported(void)
return false;
}
+static inline boo
[PATCH 0/3] target/arm: Add nested virtualization support
This series add support for ARMv8.3/8.4 nested virtualization support in KVM mode. It's based on Marc Zyngier's kernel KVM patches[1], and has been tested on a FVP model to run a L2 guest with Qemu. Now the feature can be enabled by "-M virt,accel=kvm,virtualization=on" when starting a VM. [1] https://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git/log/?h=kvm-arm64/nv-5.12-WIP Haibo Xu (3): Update linux header with new arm64 NV macro. Enable support for setting KVM vGIC maintenance IRQ Enable nested virtualization support in arm64 KVM mode hw/arm/virt.c | 11 --- hw/intc/arm_gicv3_common.c | 1 + hw/intc/arm_gicv3_kvm.c| 16 include/hw/intc/arm_gicv3_common.h | 1 + linux-headers/asm-arm64/kvm.h | 2 ++ linux-headers/linux/kvm.h | 1 + target/arm/cpu.h | 8 target/arm/kvm64.c | 14 ++ target/arm/kvm_arm.h | 28 9 files changed, 79 insertions(+), 3 deletions(-) -- 2.17.1
[PATCH 2/3] Enable support for setting KVM vGIC maintenance IRQ
Uses the new VGIC KVM device attribute to set the maintenance IRQ.
This is fixed to use IRQ 25(PPI 9), as a platform decision matching
the arm64 SBSA recommendation.
Signed-off-by: Haibo Xu
---
hw/intc/arm_gicv3_common.c | 1 +
hw/intc/arm_gicv3_kvm.c| 16
include/hw/intc/arm_gicv3_common.h | 1 +
3 files changed, 18 insertions(+)
diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c
index 58ef65f589..3ac10c8e61 100644
--- a/hw/intc/arm_gicv3_common.c
+++ b/hw/intc/arm_gicv3_common.c
@@ -495,6 +495,7 @@ static Property arm_gicv3_common_properties[] = {
DEFINE_PROP_UINT32("num-irq", GICv3State, num_irq, 32),
DEFINE_PROP_UINT32("revision", GICv3State, revision, 3),
DEFINE_PROP_BOOL("has-security-extensions", GICv3State, security_extn, 0),
+DEFINE_PROP_BOOL("has-virtualization-extensions", GICv3State, virt_extn,
0),
DEFINE_PROP_ARRAY("redist-region-count", GICv3State, nb_redist_regions,
redist_region_count, qdev_prop_uint32, uint32_t),
DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
index 65a4c880a3..1e1ca66e2c 100644
--- a/hw/intc/arm_gicv3_kvm.c
+++ b/hw/intc/arm_gicv3_kvm.c
@@ -826,6 +826,22 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error
**errp)
kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true, _abort);
+if (s->virt_extn) {
+bool maint_irq_allowed;
+uint32_t maint_irq = 25;
+
+maint_irq_allowed =
+kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ,
0);
+if (!maint_irq_allowed) {
+error_setg(errp, "VGICv3 setting maintenance IRQ are not "
+ "supported by this host kernel");
+return;
+}
+
+kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ,
+ 0, _irq, true, _abort);
+}
+
kvm_arm_register_device(>iomem_dist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
KVM_VGIC_V3_ADDR_TYPE_DIST, s->dev_fd, 0);
diff --git a/include/hw/intc/arm_gicv3_common.h
b/include/hw/intc/arm_gicv3_common.h
index 91491a2f66..921ddc2c5f 100644
--- a/include/hw/intc/arm_gicv3_common.h
+++ b/include/hw/intc/arm_gicv3_common.h
@@ -220,6 +220,7 @@ struct GICv3State {
uint32_t num_irq;
uint32_t revision;
bool security_extn;
+bool virt_extn;
bool irq_reset_nonsecure;
bool gicd_no_migration_shift_bug;
--
2.17.1
Re: [RFC PATCH v2 4/5] Add migration support for KVM guest with MTE
On Thu, 18 Mar 2021 at 04:11, Dr. David Alan Gilbert
wrote:
>
> * Haibo Xu (haibo...@linaro.org) wrote:
> > To make it easier to keep the page tags sync with
> > the page data, tags for one page are appended to
> > the data during ram save iteration.
> >
> > This patch only add the pre-copy migration support.
> > Post-copy and compress as well as zero page saving
> > are not supported yet.
> >
> > Signed-off-by: Haibo Xu
>
> My guess is that this doesn't work with a lot of other options; e.g.
> postcopy and probably compression and a bunch of other things.
> Postcopy I can see you'll need some interesting kernel changes for -
> you'd need to be able to atomically place a page with it's tag data.
>
> You probably need to add stuff to migrate_caps_check to disable
> features that you don't support.
>
Hi David,
Thanks so much for the comments!
You are right, this RFC patch only supports pre-copy mode, no
postcopy, no compression.
As a RFC, here just want to finalize the tag migration process, that is:
1. let the tag go with the page data(the current choice) which may be
a little complex to put
them into the current migration process.
2. migrate them separately which is easy to implement with the current
migration(treat the tags
as device status), but it would be hard to keep the page data and
tag to sync with each other.
3. Any other ways?
Once the tag migration process is finalized, a new formal patch series
with postcopy as well as
compression should be reworked.
What's more, you mentioned that "some interesting kernel changes are
needed to atomically
place a page with it's tag data". You mean a single kernel API to
store page data and tag in
the migration load process?
Regards,
Haibo
> > ---
> > include/hw/arm/virt.h| 2 +
> > include/migration/misc.h | 1 +
> > migration/ram.c | 86 +++-
> > 3 files changed, 88 insertions(+), 1 deletion(-)
> >
> > diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
> > index 921416f918..8b28cde8bf 100644
> > --- a/include/hw/arm/virt.h
> > +++ b/include/hw/arm/virt.h
> > @@ -166,6 +166,8 @@ struct VirtMachineState {
> > PCIBus *bus;
> > char *oem_id;
> > char *oem_table_id;
> > +/* migrate memory tags */
> > +NotifierWithReturn precopy_notifier;
> > };
> >
> > #define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM)
> > diff --git a/include/migration/misc.h b/include/migration/misc.h
> > index bccc1b6b44..005133f471 100644
> > --- a/include/migration/misc.h
> > +++ b/include/migration/misc.h
> > @@ -38,6 +38,7 @@ void precopy_add_notifier(NotifierWithReturn *n);
> > void precopy_remove_notifier(NotifierWithReturn *n);
> > int precopy_notify(PrecopyNotifyReason reason, Error **errp);
> > void precopy_enable_free_page_optimization(void);
> > +void precopy_enable_metadata_migration(void);
> >
> > void ram_mig_init(void);
> > void qemu_guest_free_page_hint(void *addr, size_t len);
> > diff --git a/migration/ram.c b/migration/ram.c
> > index 72143da0ac..e67b798c3b 100644
> > --- a/migration/ram.c
> > +++ b/migration/ram.c
> > @@ -53,10 +53,12 @@
> > #include "block.h"
> > #include "sysemu/sysemu.h"
> > #include "sysemu/cpu-throttle.h"
> > +#include "sysemu/kvm.h"
> > #include "savevm.h"
> > #include "qemu/iov.h"
> > #include "multifd.h"
> > #include "sysemu/runstate.h"
> > +#include "kvm_arm.h"
> >
> > #if defined(__linux__)
> > #include "qemu/userfaultfd.h"
> > @@ -80,6 +82,9 @@
> > #define RAM_SAVE_FLAG_XBZRLE 0x40
> > /* 0x80 is reserved in migration.h start with 0x100 next */
> > #define RAM_SAVE_FLAG_COMPRESS_PAGE0x100
> > +#define RAM_SAVE_FLAG_MTE 0x200
>
> I think that's using the last free bit in the flags field, which is a
> bit painful.
>
Yes, only 0x80(reserved) and 0x200 are available.
> > +#define MTE_GRANULE_SIZE (16)
> >
> > static inline bool is_zero_range(uint8_t *p, uint64_t size)
> > {
> > @@ -317,6 +322,8 @@ struct RAMState {
> > bool ram_bulk_stage;
> > /* The free page optimization is enabled */
> > bool fpo_enabled;
> > +/* The RAM meta data(e.t memory tag) is enabled */
> > +bool metadata_enabled;
> > /* How many times we have dirty too many pages */
> > int dirty_rate_high_cnt;
> > /* these variables are used for bitmap sync */
> > @@ -394,6 +401,15 @@
[RFC PATCH v2 3/5] Add APIs to get/set MTE tags
MTE spec provide instructions to retrieve the memory tags:
(1) LDG, at 16 bytes granularity, and available in both user
and kernel space;
(2) LDGM, at 256 bytes granularity in maximum, and only
available in kernel space
To improve the performance, KVM has exposed the LDGM capability
to user space by providing a new APIs. This patch is just a
wrapper for the KVM APIs.
Signed-off-by: Haibo Xu
---
target/arm/kvm64.c | 24
target/arm/kvm_arm.h | 2 ++
2 files changed, 26 insertions(+)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 73a191f8e1..3157025316 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -1606,3 +1606,27 @@ bool kvm_arm_verify_ext_dabt_pending(CPUState *cs)
}
return false;
}
+
+int kvm_arm_mte_get_tags(uint64_t ipa, uint64_t len, uint8_t *buf)
+{
+struct kvm_arm_copy_mte_tags args = {
+.guest_ipa = ipa,
+.length = len,
+.addr = buf,
+.flags = KVM_ARM_TAGS_FROM_GUEST,
+};
+
+return kvm_vm_ioctl(kvm_state, KVM_ARM_MTE_COPY_TAGS, );
+}
+
+int kvm_arm_mte_set_tags(uint64_t ipa, uint64_t len, uint8_t *buf)
+{
+struct kvm_arm_copy_mte_tags args = {
+.guest_ipa = ipa,
+.length = len,
+.addr = buf,
+.flags = KVM_ARM_TAGS_TO_GUEST,
+};
+
+return kvm_vm_ioctl(kvm_state, KVM_ARM_MTE_COPY_TAGS, );
+}
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index 34f8daa377..bbb833d6c6 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -360,6 +360,8 @@ int kvm_arm_vgic_probe(void);
void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
void kvm_arm_pmu_init(CPUState *cs);
+int kvm_arm_mte_get_tags(uint64_t ipa, uint64_t len, uint8_t *buf);
+int kvm_arm_mte_set_tags(uint64_t ipa, uint64_t len, uint8_t *buf);
/**
* kvm_arm_pvtime_init:
--
2.17.1
[RFC PATCH v2 5/5] Enable the MTE support for KVM guest
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 22 +++---
1 file changed, 19 insertions(+), 3 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 76658b93a3..36cfdb29e9 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -79,6 +79,7 @@
#include "hw/virtio/virtio-iommu.h"
#include "hw/char/pl011.h"
#include "qemu/guest-random.h"
+#include "migration/misc.h"
#define DEFINE_VIRT_MACHINE_LATEST(major, minor, latest) \
static void virt_##major##_##minor##_class_init(ObjectClass *oc, \
@@ -828,6 +829,21 @@ static void virt_powerdown_req(Notifier *n, void *opaque)
}
}
+static int virt_precopy_notify(NotifierWithReturn *n, void *data)
+{
+PrecopyNotifyData *pnd = data;
+
+switch (pnd->reason) {
+case PRECOPY_NOTIFY_SETUP:
+precopy_enable_metadata_migration();
+break;
+default:
+break;
+}
+
+return 0;
+}
+
static void create_gpio_keys(char *fdt, DeviceState *pl061_dev,
uint32_t phandle)
{
@@ -1912,9 +1928,9 @@ static void machvirt_init(MachineState *machine)
}
if (vms->mte && kvm_enabled()) {
-error_report("mach-virt: KVM does not support providing "
- "MTE to the guest CPU");
-exit(1);
+/* connect migration precopy request */
+vms->precopy_notifier.notify = virt_precopy_notify;
+precopy_add_notifier(>precopy_notifier);
}
create_fdt(vms);
--
2.17.1
[RFC PATCH v2 4/5] Add migration support for KVM guest with MTE
To make it easier to keep the page tags sync with
the page data, tags for one page are appended to
the data during ram save iteration.
This patch only add the pre-copy migration support.
Post-copy and compress as well as zero page saving
are not supported yet.
Signed-off-by: Haibo Xu
---
include/hw/arm/virt.h| 2 +
include/migration/misc.h | 1 +
migration/ram.c | 86 +++-
3 files changed, 88 insertions(+), 1 deletion(-)
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index 921416f918..8b28cde8bf 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -166,6 +166,8 @@ struct VirtMachineState {
PCIBus *bus;
char *oem_id;
char *oem_table_id;
+/* migrate memory tags */
+NotifierWithReturn precopy_notifier;
};
#define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM)
diff --git a/include/migration/misc.h b/include/migration/misc.h
index bccc1b6b44..005133f471 100644
--- a/include/migration/misc.h
+++ b/include/migration/misc.h
@@ -38,6 +38,7 @@ void precopy_add_notifier(NotifierWithReturn *n);
void precopy_remove_notifier(NotifierWithReturn *n);
int precopy_notify(PrecopyNotifyReason reason, Error **errp);
void precopy_enable_free_page_optimization(void);
+void precopy_enable_metadata_migration(void);
void ram_mig_init(void);
void qemu_guest_free_page_hint(void *addr, size_t len);
diff --git a/migration/ram.c b/migration/ram.c
index 72143da0ac..e67b798c3b 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -53,10 +53,12 @@
#include "block.h"
#include "sysemu/sysemu.h"
#include "sysemu/cpu-throttle.h"
+#include "sysemu/kvm.h"
#include "savevm.h"
#include "qemu/iov.h"
#include "multifd.h"
#include "sysemu/runstate.h"
+#include "kvm_arm.h"
#if defined(__linux__)
#include "qemu/userfaultfd.h"
@@ -80,6 +82,9 @@
#define RAM_SAVE_FLAG_XBZRLE 0x40
/* 0x80 is reserved in migration.h start with 0x100 next */
#define RAM_SAVE_FLAG_COMPRESS_PAGE0x100
+#define RAM_SAVE_FLAG_MTE 0x200
+
+#define MTE_GRANULE_SIZE (16)
static inline bool is_zero_range(uint8_t *p, uint64_t size)
{
@@ -317,6 +322,8 @@ struct RAMState {
bool ram_bulk_stage;
/* The free page optimization is enabled */
bool fpo_enabled;
+/* The RAM meta data(e.t memory tag) is enabled */
+bool metadata_enabled;
/* How many times we have dirty too many pages */
int dirty_rate_high_cnt;
/* these variables are used for bitmap sync */
@@ -394,6 +401,15 @@ void precopy_enable_free_page_optimization(void)
ram_state->fpo_enabled = true;
}
+void precopy_enable_metadata_migration(void)
+{
+if (!ram_state) {
+return;
+}
+
+ram_state->metadata_enabled = true;
+}
+
uint64_t ram_bytes_remaining(void)
{
return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
@@ -1134,6 +1150,61 @@ static bool control_save_page(RAMState *rs, RAMBlock
*block, ram_addr_t offset,
return true;
}
+static int save_normal_page_mte_tags(QEMUFile *f, uint8_t *addr)
+{
+uint8_t *tag_buf = NULL;
+uint64_t ipa;
+int size = TARGET_PAGE_SIZE / MTE_GRANULE_SIZE;
+
+if (kvm_physical_memory_addr_from_host(kvm_state, addr, )) {
+/* Buffer for the page tags(one byte per tag) */
+tag_buf = g_try_malloc0(size);
+if (!tag_buf) {
+error_report("%s: Error allocating MTE tag_buf", __func__);
+return 0;
+}
+
+if (kvm_arm_mte_get_tags(ipa, TARGET_PAGE_SIZE, tag_buf) < 0) {
+error_report("%s: Can't get MTE tags from guest", __func__);
+g_free(tag_buf);
+return 0;
+}
+
+qemu_put_buffer(f, tag_buf, size);
+
+g_free(tag_buf);
+
+return size;
+}
+
+return 0;
+}
+
+static void load_normal_page_mte_tags(QEMUFile *f, uint8_t *addr)
+{
+uint8_t *tag_buf = NULL;
+uint64_t ipa;
+int size = TARGET_PAGE_SIZE / MTE_GRANULE_SIZE;
+
+if (kvm_physical_memory_addr_from_host(kvm_state, addr, )) {
+/* Buffer for the page tags(one byte per tag) */
+tag_buf = g_try_malloc0(size);
+if (!tag_buf) {
+error_report("%s: Error allocating MTE tag_buf", __func__);
+return;
+}
+
+qemu_get_buffer(f, tag_buf, size);
+if (kvm_arm_mte_set_tags(ipa, TARGET_PAGE_SIZE, tag_buf) < 0) {
+error_report("%s: Can't set MTE tags to guest", __func__);
+}
+
+g_free(tag_buf);
+}
+
+return;
+}
+
/*
* directly send the page to the stream
*
@@ -1148,6 +1219,10 @@ static bool control_save_page(RAMState *rs, RAMBlock
*block, ram_addr_t offset,
static int save_normal_page(RAMState *rs, RAMBlock *block, ram_addr_t offset,
uint8_t *buf, bool a
[RFC PATCH v2 2/5] Add basic MTE support to KVM guest
Enable the virt machine feature "mte" to work with
KVM guest. This feature is still hiden from the user
in this patch, and will be available in a later patch.
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 22 +++---
target/arm/cpu.c | 2 +-
target/arm/kvm.c | 9 +
target/arm/kvm64.c | 7 +++
4 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index aa2bbd14e0..76658b93a3 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -1988,18 +1988,18 @@ static void machvirt_init(MachineState *machine)
}
if (vms->mte) {
+/*
+ * The property exists only if MemTag is supported.
+ * If it is, we must allocate the ram to back that up.
+ */
+if (!object_property_find(cpuobj, "tag-memory")) {
+error_report("MTE requested, but not supported "
+ "by the guest CPU");
+exit(1);
+}
+
/* Create the memory region only once, but link to all cpus. */
-if (!tag_sysmem) {
-/*
- * The property exists only if MemTag is supported.
- * If it is, we must allocate the ram to back that up.
- */
-if (!object_property_find(cpuobj, "tag-memory")) {
-error_report("MTE requested, but not supported "
- "by the guest CPU");
-exit(1);
-}
-
+if (!tag_sysmem && !kvm_enabled()) {
tag_sysmem = g_new(MemoryRegion, 1);
memory_region_init(tag_sysmem, OBJECT(machine),
"tag-memory", UINT64_MAX / 32);
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index ae04884408..47bf817b61 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1847,7 +1847,7 @@ static void arm_cpu_realizefn(DeviceState *dev, Error
**errp)
cpu->secure_memory);
}
-if (cpu->tag_memory != NULL) {
+if (cpu->tag_memory != NULL && !kvm_enabled()) {
cpu_address_space_init(cs, ARMASIdx_TagNS, "cpu-tag-memory",
cpu->tag_memory);
if (has_secure) {
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index d8381ba224..3403e621ac 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -32,6 +32,7 @@
#include "hw/boards.h"
#include "hw/irq.h"
#include "qemu/log.h"
+#include "hw/arm/virt.h"
const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
KVM_CAP_LAST_INFO
@@ -274,6 +275,14 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
}
}
+if (kvm_check_extension(s, KVM_CAP_ARM_MTE) &&
+object_dynamic_cast(OBJECT(ms), TYPE_VIRT_MACHINE) &&
+VIRT_MACHINE(ms)->mte) {
+if (kvm_vm_enable_cap(s, KVM_CAP_ARM_MTE, 0)) {
+error_report("Failed to enable KVM_CAP_ARM_MTE cap");
+}
+}
+
return ret;
}
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index dff85f6db9..73a191f8e1 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -500,6 +500,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
*/
int fdarray[3];
bool sve_supported;
+bool mte_supported;
uint64_t features = 0;
uint64_t t;
int err;
@@ -646,6 +647,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
}
sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) >
0;
+mte_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_MTE) >
0;
kvm_arm_destroy_scratch_host_vcpu(fdarray);
@@ -659,6 +661,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
*ahcf)
t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
ahcf->isar.id_aa64pfr0 = t;
}
+if (mte_supported) {
+t = ahcf->isar.id_aa64pfr1;
+t = FIELD_DP64(t, ID_AA64PFR1, MTE, 2);
+ahcf->isar.id_aa64pfr1 = t;
+}
/*
* We can assume any KVM supporting CPU is at least a v8
--
2.17.1
[RFC PATCH v2 1/5] Update Linux headers with new MTE support
Signed-off-by: Haibo Xu
---
linux-headers/linux/kvm.h | 16
1 file changed, 16 insertions(+)
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 020b62a619..941743b3a7 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1056,6 +1056,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
#define KVM_CAP_SYS_HYPERV_CPUID 191
#define KVM_CAP_DIRTY_LOG_RING 192
+#define KVM_CAP_ARM_MTE 195
#ifdef KVM_CAP_IRQ_ROUTING
@@ -1241,6 +1242,20 @@ struct kvm_arm_device_addr {
__u64 addr;
};
+#define KVM_ARM_TAGS_TO_GUEST 0
+#define KVM_ARM_TAGS_FROM_GUEST 1
+
+struct kvm_arm_copy_mte_tags {
+ __u64 guest_ipa;
+ __u64 length;
+ union {
+ void *addr;
+ __u64 padding;
+ };
+ __u64 flags;
+ __u64 reserved[2];
+};
+
/*
* Device control API, available with KVM_CAP_DEVICE_CTRL
*/
@@ -1396,6 +1411,7 @@ struct kvm_s390_ucas_mapping {
/* Available with KVM_CAP_PMU_EVENT_FILTER */
#define KVM_SET_PMU_EVENT_FILTER _IOW(KVMIO, 0xb2, struct
kvm_pmu_event_filter)
#define KVM_PPC_SVM_OFF _IO(KVMIO, 0xb3)
+#define KVM_ARM_MTE_COPY_TAGS_IOR(KVMIO, 0xb4, struct
kvm_arm_copy_mte_tags)
/* ioctl for vm fd */
#define KVM_CREATE_DEVICE_IOWR(KVMIO, 0xe0, struct kvm_create_device)
--
2.17.1
[RFC PATCH v2 0/5] target/arm: Add MTE support to KVM guest
This version is rebased against kernel patches v10, and only header files are updated since v1[1]. This series add support for MTE(Memory Tagging Extension)[2] in KVM guest. It's based on Steven Price's kernel KVM patches v10[3], and has been tested to ensure that test case[4] can be passed in a KVM guest. Basic pre-copy migration test also passed between two MTE enabled kvm guest. This is a RFC patch series becuase: (1) Need to add some arm MTE specific codes to the ram migration loop. There may be better way to do that in a more abstract way. (2) Only pre-copy migration is supported and tested currently, post-copy as well as compress/zero page migration are still WIP. All kinds of feedbacks are very welcomed, especially for the migration support. Note: (1) To support MTE migration, tags for one page are appended to the page data during ram save iteration which make it easier to sync the page data and tags. [1] https://lore.kernel.org/qemu-devel/cover.1612747873.git.haibo...@linaro.org/ [2] https://community.arm.com/developer/ip-products/processors/b/ processors-ip-blog/posts/enhancing-memory-safety [3] https://patchew.org/QEMU/20210312151902.17853-1-steven.pr...@arm.com/ [4] https://elixir.bootlin.com/linux/latest/source/Documentation/ arm64/memory-tagging-extension.rst Haibo Xu (5): Update Linux headers with new MTE support Add basic MTE support to KVM guest Add APIs to get/set MTE tags Add migration support for KVM guest with MTE Enable the MTE support for KVM guest hw/arm/virt.c | 44 +--- include/hw/arm/virt.h | 2 + include/migration/misc.h | 1 + linux-headers/linux/kvm.h | 16 migration/ram.c | 86 ++- target/arm/cpu.c | 2 +- target/arm/kvm.c | 9 target/arm/kvm64.c| 31 ++ target/arm/kvm_arm.h | 2 + 9 files changed, 177 insertions(+), 16 deletions(-) -- 2.17.1
Re: [RFC PATCH 4/5] Add migration support for KVM guest with MTE
++ more migration experts! On Tue, 23 Feb 2021 at 06:47, Richard Henderson wrote: > > On 2/22/21 1:46 AM, Haibo Xu wrote: > > As I mentioned in the cover later, the reason to let the tag go with the > > memory data together is to make it easier to sync with each other. I think > > if we migratie them separately, it would be hard to keep the tags to sync > > with the data. > Well, maybe, maybe not. See below. > > > > Saying if we migration all the data first, then the tags. If the data got > > dirty during the migration of the tag memory, we may need to send the data > > again, or freeze the source VM after data migration? What's more, the > > KVM_GET_DIRTY_LOG API may not be able to differentiate between a tag and > > data changes. > I would certainly expect KVM_GET_DIRTY_LOG to only care about the normal > memory. That is, pages as viewed by the guest. > > I would expect the separate tag_memory block to be private to the host. If a > normal page is dirty, then we would read the tags into the tag_memory and > manually mark that dirty. Migration would continue as normal, and eventually > both normal and tag memory would all be clean and migrated. > > But I'll admit that it does require that we retain a buffer 1/16 the size of > main memory, which is otherwise unused, and thus this is less than ideal. So > if we do it your way, we should arrange for tcg to migrate the tag data in the > same way. > > I'll still wait for migration experts, of which I am not one. > > > r~
Re: [RFC PATCH 3/5] Add APIs to get/set MTE tags
++ more migration experts!
On Mon, 8 Feb 2021 at 11:20, Haibo Xu wrote:
>
> MTE spec provide instructions to retrieve the memory tags:
> (1) LDG, at 16 bytes granularity, and available in both user
> and kernel space;
> (2) LDGM, at 256 bytes granularity in maximum, and only
> available in kernel space
>
> To improve the performance, KVM has exposed the LDGM capability
> to user space by providing a new APIs. This patch is just a
> wrapper for the KVM APIs.
>
> Signed-off-by: Haibo Xu
> ---
> target/arm/kvm64.c | 24
> target/arm/kvm_arm.h | 2 ++
> 2 files changed, 26 insertions(+)
>
> diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> index 23f34034db..4a6790d53b 100644
> --- a/target/arm/kvm64.c
> +++ b/target/arm/kvm64.c
> @@ -1608,3 +1608,27 @@ bool kvm_arm_verify_ext_dabt_pending(CPUState *cs)
> }
> return false;
> }
> +
> +int kvm_arm_mte_get_tags(uint64_t ipa, uint64_t len, uint8_t *buf)
> +{
> +struct kvm_arm_copy_mte_tags args = {
> +.guest_ipa = ipa,
> +.length = len,
> +.addr = buf,
> +.flags = KVM_ARM_TAGS_FROM_GUEST,
> +};
> +
> +return kvm_vm_ioctl(kvm_state, KVM_ARM_MTE_COPY_TAGS, );
> +}
> +
> +int kvm_arm_mte_set_tags(uint64_t ipa, uint64_t len, uint8_t *buf)
> +{
> +struct kvm_arm_copy_mte_tags args = {
> +.guest_ipa = ipa,
> +.length = len,
> +.addr = buf,
> +.flags = KVM_ARM_TAGS_TO_GUEST,
> +};
> +
> +return kvm_vm_ioctl(kvm_state, KVM_ARM_MTE_COPY_TAGS, );
> +}
> diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
> index eb81b7059e..1b94dbe7c8 100644
> --- a/target/arm/kvm_arm.h
> +++ b/target/arm/kvm_arm.h
> @@ -358,6 +358,8 @@ int kvm_arm_vgic_probe(void);
>
> void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
> void kvm_arm_pmu_init(CPUState *cs);
> +int kvm_arm_mte_get_tags(uint64_t ipa, uint64_t len, uint8_t *buf);
> +int kvm_arm_mte_set_tags(uint64_t ipa, uint64_t len, uint8_t *buf);
>
> /**
> * kvm_arm_pvtime_init:
> --
> 2.17.1
>
Re: [RFC PATCH 5/5] Enable the MTE support for KVM guest
++ more migration experts!
On Mon, 8 Feb 2021 at 11:20, Haibo Xu wrote:
>
> Signed-off-by: Haibo Xu
> ---
> hw/arm/virt.c | 22 +++---
> 1 file changed, 19 insertions(+), 3 deletions(-)
>
> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> index 623d5e9397..c2358cf4c5 100644
> --- a/hw/arm/virt.c
> +++ b/hw/arm/virt.c
> @@ -79,6 +79,7 @@
> #include "hw/virtio/virtio-iommu.h"
> #include "hw/char/pl011.h"
> #include "qemu/guest-random.h"
> +#include "migration/misc.h"
>
> #define DEFINE_VIRT_MACHINE_LATEST(major, minor, latest) \
> static void virt_##major##_##minor##_class_init(ObjectClass *oc, \
> @@ -821,6 +822,21 @@ static void virt_powerdown_req(Notifier *n, void *opaque)
> }
> }
>
> +static int virt_precopy_notify(NotifierWithReturn *n, void *data)
> +{
> +PrecopyNotifyData *pnd = data;
> +
> +switch (pnd->reason) {
> +case PRECOPY_NOTIFY_SETUP:
> +precopy_enable_metadata_migration();
> +break;
> +default:
> +break;
> +}
> +
> +return 0;
> +}
> +
> static void create_gpio_keys(const VirtMachineState *vms,
> DeviceState *pl061_dev,
> uint32_t phandle)
> @@ -1898,9 +1914,9 @@ static void machvirt_init(MachineState *machine)
> }
>
> if (vms->mte && kvm_enabled()) {
> -error_report("mach-virt: KVM does not support providing "
> - "MTE to the guest CPU");
> -exit(1);
> +/* connect migration precopy request */
> +vms->precopy_notifier.notify = virt_precopy_notify;
> +precopy_add_notifier(>precopy_notifier);
> }
>
> create_fdt(vms);
> --
> 2.17.1
>
Re: [RFC PATCH 2/5] Add basic MTE support to KVM guest
++ more migration experts!
On Mon, 8 Feb 2021 at 11:20, Haibo Xu wrote:
>
> Enable the virt machine feature "mte" to work with
> KVM guest. This feature is still hiden from the user
> in this patch, and will be available in a later patch.
>
> Signed-off-by: Haibo Xu
> ---
> hw/arm/virt.c | 22 +++---
> target/arm/cpu.c | 2 +-
> target/arm/kvm.c | 9 +
> target/arm/kvm64.c | 7 +++
> 4 files changed, 28 insertions(+), 12 deletions(-)
>
> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> index 399da73454..623d5e9397 100644
> --- a/hw/arm/virt.c
> +++ b/hw/arm/virt.c
> @@ -1974,18 +1974,18 @@ static void machvirt_init(MachineState *machine)
> }
>
> if (vms->mte) {
> +/*
> + * The property exists only if MemTag is supported.
> + * If it is, we must allocate the ram to back that up.
> + */
> +if (!object_property_find(cpuobj, "tag-memory")) {
> +error_report("MTE requested, but not supported "
> + "by the guest CPU");
> +exit(1);
> +}
> +
> /* Create the memory region only once, but link to all cpus. */
> -if (!tag_sysmem) {
> -/*
> - * The property exists only if MemTag is supported.
> - * If it is, we must allocate the ram to back that up.
> - */
> -if (!object_property_find(cpuobj, "tag-memory")) {
> -error_report("MTE requested, but not supported "
> - "by the guest CPU");
> -exit(1);
> -}
> -
> +if (!tag_sysmem && !kvm_enabled()) {
> tag_sysmem = g_new(MemoryRegion, 1);
> memory_region_init(tag_sysmem, OBJECT(machine),
> "tag-memory", UINT64_MAX / 32);
> diff --git a/target/arm/cpu.c b/target/arm/cpu.c
> index 40142ac141..50f3223944 100644
> --- a/target/arm/cpu.c
> +++ b/target/arm/cpu.c
> @@ -1831,7 +1831,7 @@ static void arm_cpu_realizefn(DeviceState *dev, Error
> **errp)
> cpu->secure_memory);
> }
>
> -if (cpu->tag_memory != NULL) {
> +if (cpu->tag_memory != NULL && !kvm_enabled()) {
> cpu_address_space_init(cs, ARMASIdx_TagNS, "cpu-tag-memory",
> cpu->tag_memory);
> if (has_secure) {
> diff --git a/target/arm/kvm.c b/target/arm/kvm.c
> index ffe186de8d..33630b2b70 100644
> --- a/target/arm/kvm.c
> +++ b/target/arm/kvm.c
> @@ -32,6 +32,7 @@
> #include "hw/boards.h"
> #include "hw/irq.h"
> #include "qemu/log.h"
> +#include "hw/arm/virt.h"
>
> const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
> KVM_CAP_LAST_INFO
> @@ -272,6 +273,14 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
> }
> }
>
> +if (kvm_check_extension(s, KVM_CAP_ARM_MTE) &&
> +object_dynamic_cast(OBJECT(ms), TYPE_VIRT_MACHINE) &&
> +VIRT_MACHINE(ms)->mte) {
> +if (kvm_vm_enable_cap(s, KVM_CAP_ARM_MTE, 0)) {
> +error_report("Failed to enable KVM_CAP_ARM_MTE cap");
> +}
> +}
> +
> return ret;
> }
>
> diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> index 3c37fc4fb6..23f34034db 100644
> --- a/target/arm/kvm64.c
> +++ b/target/arm/kvm64.c
> @@ -500,6 +500,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> *ahcf)
> */
> int fdarray[3];
> bool sve_supported;
> +bool mte_supported;
> uint64_t features = 0;
> uint64_t t;
> int err;
> @@ -646,6 +647,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> *ahcf)
> }
>
> sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE)
> > 0;
> +mte_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_MTE)
> > 0;
>
> kvm_arm_destroy_scratch_host_vcpu(fdarray);
>
> @@ -659,6 +661,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> *ahcf)
> t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
> ahcf->isar.id_aa64pfr0 = t;
> }
> +if (mte_supported) {
> +t = ahcf->isar.id_aa64pfr1;
> +t = FIELD_DP64(t, ID_AA64PFR1, MTE, 2);
> +ahcf->isar.id_aa64pfr1 = t;
> +}
>
> /*
> * We can assume any KVM supporting CPU is at least a v8
> --
> 2.17.1
>
Re: [RFC PATCH 1/5] Update Linux headers with new MTE support
++ more migration experts!
On Mon, 8 Feb 2021 at 11:20, Haibo Xu wrote:
>
> Signed-off-by: Haibo Xu
> ---
> linux-headers/linux/kvm.h | 15 +++
> 1 file changed, 15 insertions(+)
>
> diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
> index 020b62a619..6a291a9a35 100644
> --- a/linux-headers/linux/kvm.h
> +++ b/linux-headers/linux/kvm.h
> @@ -1056,6 +1056,7 @@ struct kvm_ppc_resize_hpt {
> #define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
> #define KVM_CAP_SYS_HYPERV_CPUID 191
> #define KVM_CAP_DIRTY_LOG_RING 192
> +#define KVM_CAP_ARM_MTE 193
>
> #ifdef KVM_CAP_IRQ_ROUTING
>
> @@ -1241,6 +1242,19 @@ struct kvm_arm_device_addr {
> __u64 addr;
> };
>
> +#define KVM_ARM_TAGS_TO_GUEST 0
> +#define KVM_ARM_TAGS_FROM_GUEST 1
> +
> +struct kvm_arm_copy_mte_tags {
> + __u64 guest_ipa;
> + __u64 length;
> + union {
> + void *addr;
> + __u64 padding;
> + };
> + __u64 flags;
> +};
> +
> /*
> * Device control API, available with KVM_CAP_DEVICE_CTRL
> */
> @@ -1396,6 +1410,7 @@ struct kvm_s390_ucas_mapping {
> /* Available with KVM_CAP_PMU_EVENT_FILTER */
> #define KVM_SET_PMU_EVENT_FILTER _IOW(KVMIO, 0xb2, struct
> kvm_pmu_event_filter)
> #define KVM_PPC_SVM_OFF _IO(KVMIO, 0xb3)
> +#define KVM_ARM_MTE_COPY_TAGS_IOR(KVMIO, 0xb4, struct
> kvm_arm_copy_mte_tags)
>
> /* ioctl for vm fd */
> #define KVM_CREATE_DEVICE_IOWR(KVMIO, 0xe0, struct
> kvm_create_device)
> --
> 2.17.1
>
Re: [RFC PATCH 4/5] Add migration support for KVM guest with MTE
On Tue, 16 Feb 2021 at 23:31, Richard Henderson
wrote:
>
> On 2/7/21 7:20 PM, Haibo Xu wrote:
> > +if (kvm_physical_memory_addr_from_host(kvm_state, addr, )) {
> > +/* Buffer for the page tags(one byte per tag) */
> > +tag_buf = g_try_malloc0(size);
> > +if (!tag_buf) {
> > +error_report("%s: Error allocating MTE tag_buf", __func__);
> > +return 0;
> > +}
> > +
> > +if (kvm_arm_mte_get_tags(ipa, TARGET_PAGE_SIZE, tag_buf) < 0) {
> > +error_report("%s: Can't get MTE tags from guest", __func__);
> > +g_free(tag_buf);
> > +return 0;
> > +}
> > +
> > +qemu_put_buffer(f, tag_buf, size);
> > +
> > +g_free(tag_buf);
> > +
> > +return size;
> > +}
>
> So, in patch 2 you disabled the allocation of tag-memory. Now you're
> allocating new memory (admittedly quite a small amount -- 1/16th of a page,
> small enough to just be a local variable).
>
Hi Richard!
Thanks so much for the comments!
Yes, the allocated memory here is for temporary tag storage. As you
mentioned, it can be
defined as a local variable which is reserved for temporary tag buffers.
> Why don't you allocate tag-memory, copy the data into it, and then let
> migration proceed as normal. Then you don't have to have a random data block
> that happens to follow each ram page.
>
As I mentioned in the cover later, the reason to let the tag go with
the memory data together
is to make it easier to sync with each other. I think if we migratie
them separately, it would be
hard to keep the tags to sync with the data.
Saying if we migration all the data first, then the tags. If the data
got dirty during the migration
of the tag memory, we may need to send the data again, or freeze the
source VM after data
migration? What's more, the KVM_GET_DIRTY_LOG API may not be able to
differentiate
between a tag and data changes.
I'm not sure whether the aforementioned situation does exist. Please
correct me if something goes wrong!
> I'm concerned that what you're doing here makes it impossible to migrate
> between kvm and tcg.
>
>
You mean to migrate from a KVM mode VM to a TCG mode VM?
> r~
Regards,
Haibo
Re: [RFC PATCH 0/5] target/arm: Add MTE support to KVM guest
On Tue, 16 Feb 2021 at 19:20, Peter Maydell wrote: > > Adding the migration maintainers to the cc list, because > the real meat of this patchset is "how should the migration > of MTE tags be integrated into the migration/ram.c code", > which is pretty well out of my area of expertise... > > thanks > -- PMM > Hi Peter, Thanks for cc-ing to the related maintainers! Regards, Haibo > On Mon, 8 Feb 2021 at 03:20, Haibo Xu wrote: > > > > This series add support for MTE(Memory Tagging Extension)[1] > > in KVM guest. It's based on Steven Price's kernel KVM patches > > V7[2], and has been tested to ensure that test case[3] can be > > passed in a KVM guest. Basic pre-copy migration test also passed > > between two MTE enabled kvm guest. > > > > This is a RFC patch series becuase: > > (1) Need to add some arm MTE specific codes to the ram migration > > loop. There may be better way to do that in a more abstract way. > > (2) Only pre-copy migration is supported and tested currently, > > post-copy as well as compress/zero page migration are still WIP. > > > > All kinds of feedbacks are very welcomed, especially for the migration > > support. > > > > Note: > > (1) To support MTE migration, tags for one page are appended to > > the page data during ram save iteration which make it easier > > to sync the page data and tags. > > > > [1] https://community.arm.com/developer/ip-products/processors/b/ > > processors-ip-blog/posts/enhancing-memory-safety > > [2] https://lwn.net/Articles/842827/ > > [3] https://elixir.bootlin.com/linux/latest/source/Documentation/ > > arm64/memory-tagging-extension.rst > > > > Haibo Xu (5): > > Update Linux headers with new MTE support > > Add basic MTE support to KVM guest > > Add APIs to get/set MTE tags > > Add migration support for KVM guest with MTE > > Enable the MTE support for KVM guest > > > > hw/arm/virt.c | 44 +--- > > include/hw/arm/virt.h | 2 + > > include/migration/misc.h | 1 + > > linux-headers/linux/kvm.h | 15 +++ > > migration/ram.c | 86 ++- > > target/arm/cpu.c | 2 +- > > target/arm/kvm.c | 9 > > target/arm/kvm64.c| 31 ++ > > target/arm/kvm_arm.h | 2 + > > 9 files changed, 176 insertions(+), 16 deletions(-)
[RFC PATCH 5/5] Enable the MTE support for KVM guest
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 22 +++---
1 file changed, 19 insertions(+), 3 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 623d5e9397..c2358cf4c5 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -79,6 +79,7 @@
#include "hw/virtio/virtio-iommu.h"
#include "hw/char/pl011.h"
#include "qemu/guest-random.h"
+#include "migration/misc.h"
#define DEFINE_VIRT_MACHINE_LATEST(major, minor, latest) \
static void virt_##major##_##minor##_class_init(ObjectClass *oc, \
@@ -821,6 +822,21 @@ static void virt_powerdown_req(Notifier *n, void *opaque)
}
}
+static int virt_precopy_notify(NotifierWithReturn *n, void *data)
+{
+PrecopyNotifyData *pnd = data;
+
+switch (pnd->reason) {
+case PRECOPY_NOTIFY_SETUP:
+precopy_enable_metadata_migration();
+break;
+default:
+break;
+}
+
+return 0;
+}
+
static void create_gpio_keys(const VirtMachineState *vms,
DeviceState *pl061_dev,
uint32_t phandle)
@@ -1898,9 +1914,9 @@ static void machvirt_init(MachineState *machine)
}
if (vms->mte && kvm_enabled()) {
-error_report("mach-virt: KVM does not support providing "
- "MTE to the guest CPU");
-exit(1);
+/* connect migration precopy request */
+vms->precopy_notifier.notify = virt_precopy_notify;
+precopy_add_notifier(>precopy_notifier);
}
create_fdt(vms);
--
2.17.1
[RFC PATCH 2/5] Add basic MTE support to KVM guest
Enable the virt machine feature "mte" to work with
KVM guest. This feature is still hiden from the user
in this patch, and will be available in a later patch.
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 22 +++---
target/arm/cpu.c | 2 +-
target/arm/kvm.c | 9 +
target/arm/kvm64.c | 7 +++
4 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 399da73454..623d5e9397 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -1974,18 +1974,18 @@ static void machvirt_init(MachineState *machine)
}
if (vms->mte) {
+/*
+ * The property exists only if MemTag is supported.
+ * If it is, we must allocate the ram to back that up.
+ */
+if (!object_property_find(cpuobj, "tag-memory")) {
+error_report("MTE requested, but not supported "
+ "by the guest CPU");
+exit(1);
+}
+
/* Create the memory region only once, but link to all cpus. */
-if (!tag_sysmem) {
-/*
- * The property exists only if MemTag is supported.
- * If it is, we must allocate the ram to back that up.
- */
-if (!object_property_find(cpuobj, "tag-memory")) {
-error_report("MTE requested, but not supported "
- "by the guest CPU");
-exit(1);
-}
-
+if (!tag_sysmem && !kvm_enabled()) {
tag_sysmem = g_new(MemoryRegion, 1);
memory_region_init(tag_sysmem, OBJECT(machine),
"tag-memory", UINT64_MAX / 32);
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 40142ac141..50f3223944 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1831,7 +1831,7 @@ static void arm_cpu_realizefn(DeviceState *dev, Error
**errp)
cpu->secure_memory);
}
-if (cpu->tag_memory != NULL) {
+if (cpu->tag_memory != NULL && !kvm_enabled()) {
cpu_address_space_init(cs, ARMASIdx_TagNS, "cpu-tag-memory",
cpu->tag_memory);
if (has_secure) {
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index ffe186de8d..33630b2b70 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -32,6 +32,7 @@
#include "hw/boards.h"
#include "hw/irq.h"
#include "qemu/log.h"
+#include "hw/arm/virt.h"
const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
KVM_CAP_LAST_INFO
@@ -272,6 +273,14 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
}
}
+if (kvm_check_extension(s, KVM_CAP_ARM_MTE) &&
+object_dynamic_cast(OBJECT(ms), TYPE_VIRT_MACHINE) &&
+VIRT_MACHINE(ms)->mte) {
+if (kvm_vm_enable_cap(s, KVM_CAP_ARM_MTE, 0)) {
+error_report("Failed to enable KVM_CAP_ARM_MTE cap");
+}
+}
+
return ret;
}
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 3c37fc4fb6..23f34034db 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -500,6 +500,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
*/
int fdarray[3];
bool sve_supported;
+bool mte_supported;
uint64_t features = 0;
uint64_t t;
int err;
@@ -646,6 +647,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
}
sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) >
0;
+mte_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_MTE) >
0;
kvm_arm_destroy_scratch_host_vcpu(fdarray);
@@ -659,6 +661,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
*ahcf)
t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
ahcf->isar.id_aa64pfr0 = t;
}
+if (mte_supported) {
+t = ahcf->isar.id_aa64pfr1;
+t = FIELD_DP64(t, ID_AA64PFR1, MTE, 2);
+ahcf->isar.id_aa64pfr1 = t;
+}
/*
* We can assume any KVM supporting CPU is at least a v8
--
2.17.1
[RFC PATCH 3/5] Add APIs to get/set MTE tags
MTE spec provide instructions to retrieve the memory tags:
(1) LDG, at 16 bytes granularity, and available in both user
and kernel space;
(2) LDGM, at 256 bytes granularity in maximum, and only
available in kernel space
To improve the performance, KVM has exposed the LDGM capability
to user space by providing a new APIs. This patch is just a
wrapper for the KVM APIs.
Signed-off-by: Haibo Xu
---
target/arm/kvm64.c | 24
target/arm/kvm_arm.h | 2 ++
2 files changed, 26 insertions(+)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 23f34034db..4a6790d53b 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -1608,3 +1608,27 @@ bool kvm_arm_verify_ext_dabt_pending(CPUState *cs)
}
return false;
}
+
+int kvm_arm_mte_get_tags(uint64_t ipa, uint64_t len, uint8_t *buf)
+{
+struct kvm_arm_copy_mte_tags args = {
+.guest_ipa = ipa,
+.length = len,
+.addr = buf,
+.flags = KVM_ARM_TAGS_FROM_GUEST,
+};
+
+return kvm_vm_ioctl(kvm_state, KVM_ARM_MTE_COPY_TAGS, );
+}
+
+int kvm_arm_mte_set_tags(uint64_t ipa, uint64_t len, uint8_t *buf)
+{
+struct kvm_arm_copy_mte_tags args = {
+.guest_ipa = ipa,
+.length = len,
+.addr = buf,
+.flags = KVM_ARM_TAGS_TO_GUEST,
+};
+
+return kvm_vm_ioctl(kvm_state, KVM_ARM_MTE_COPY_TAGS, );
+}
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index eb81b7059e..1b94dbe7c8 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -358,6 +358,8 @@ int kvm_arm_vgic_probe(void);
void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
void kvm_arm_pmu_init(CPUState *cs);
+int kvm_arm_mte_get_tags(uint64_t ipa, uint64_t len, uint8_t *buf);
+int kvm_arm_mte_set_tags(uint64_t ipa, uint64_t len, uint8_t *buf);
/**
* kvm_arm_pvtime_init:
--
2.17.1
[RFC PATCH 1/5] Update Linux headers with new MTE support
Signed-off-by: Haibo Xu
---
linux-headers/linux/kvm.h | 15 +++
1 file changed, 15 insertions(+)
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 020b62a619..6a291a9a35 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1056,6 +1056,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
#define KVM_CAP_SYS_HYPERV_CPUID 191
#define KVM_CAP_DIRTY_LOG_RING 192
+#define KVM_CAP_ARM_MTE 193
#ifdef KVM_CAP_IRQ_ROUTING
@@ -1241,6 +1242,19 @@ struct kvm_arm_device_addr {
__u64 addr;
};
+#define KVM_ARM_TAGS_TO_GUEST 0
+#define KVM_ARM_TAGS_FROM_GUEST 1
+
+struct kvm_arm_copy_mte_tags {
+ __u64 guest_ipa;
+ __u64 length;
+ union {
+ void *addr;
+ __u64 padding;
+ };
+ __u64 flags;
+};
+
/*
* Device control API, available with KVM_CAP_DEVICE_CTRL
*/
@@ -1396,6 +1410,7 @@ struct kvm_s390_ucas_mapping {
/* Available with KVM_CAP_PMU_EVENT_FILTER */
#define KVM_SET_PMU_EVENT_FILTER _IOW(KVMIO, 0xb2, struct
kvm_pmu_event_filter)
#define KVM_PPC_SVM_OFF _IO(KVMIO, 0xb3)
+#define KVM_ARM_MTE_COPY_TAGS_IOR(KVMIO, 0xb4, struct
kvm_arm_copy_mte_tags)
/* ioctl for vm fd */
#define KVM_CREATE_DEVICE_IOWR(KVMIO, 0xe0, struct kvm_create_device)
--
2.17.1
[RFC PATCH 0/5] target/arm: Add MTE support to KVM guest
This series add support for MTE(Memory Tagging Extension)[1] in KVM guest. It's based on Steven Price's kernel KVM patches V7[2], and has been tested to ensure that test case[3] can be passed in a KVM guest. Basic pre-copy migration test also passed between two MTE enabled kvm guest. This is a RFC patch series becuase: (1) Need to add some arm MTE specific codes to the ram migration loop. There may be better way to do that in a more abstract way. (2) Only pre-copy migration is supported and tested currently, post-copy as well as compress/zero page migration are still WIP. All kinds of feedbacks are very welcomed, especially for the migration support. Note: (1) To support MTE migration, tags for one page are appended to the page data during ram save iteration which make it easier to sync the page data and tags. [1] https://community.arm.com/developer/ip-products/processors/b/ processors-ip-blog/posts/enhancing-memory-safety [2] https://lwn.net/Articles/842827/ [3] https://elixir.bootlin.com/linux/latest/source/Documentation/ arm64/memory-tagging-extension.rst Haibo Xu (5): Update Linux headers with new MTE support Add basic MTE support to KVM guest Add APIs to get/set MTE tags Add migration support for KVM guest with MTE Enable the MTE support for KVM guest hw/arm/virt.c | 44 +--- include/hw/arm/virt.h | 2 + include/migration/misc.h | 1 + linux-headers/linux/kvm.h | 15 +++ migration/ram.c | 86 ++- target/arm/cpu.c | 2 +- target/arm/kvm.c | 9 target/arm/kvm64.c| 31 ++ target/arm/kvm_arm.h | 2 + 9 files changed, 176 insertions(+), 16 deletions(-) -- 2.17.1
[RFC PATCH 4/5] Add migration support for KVM guest with MTE
To make it easier to keep the page tags sync with
the page data, tags for one page are appended to
the data during ram save iteration.
This patch only add the pre-copy migration support.
Post-copy and compress as well as zero page saving
are not supported yet.
Signed-off-by: Haibo Xu
---
include/hw/arm/virt.h| 2 +
include/migration/misc.h | 1 +
migration/ram.c | 86 +++-
3 files changed, 88 insertions(+), 1 deletion(-)
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index 36fcb29641..6182b3e1d1 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -165,6 +165,8 @@ struct VirtMachineState {
DeviceState *acpi_dev;
Notifier powerdown_notifier;
PCIBus *bus;
+/* migrate memory tags */
+NotifierWithReturn precopy_notifier;
};
#define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM)
diff --git a/include/migration/misc.h b/include/migration/misc.h
index bccc1b6b44..005133f471 100644
--- a/include/migration/misc.h
+++ b/include/migration/misc.h
@@ -38,6 +38,7 @@ void precopy_add_notifier(NotifierWithReturn *n);
void precopy_remove_notifier(NotifierWithReturn *n);
int precopy_notify(PrecopyNotifyReason reason, Error **errp);
void precopy_enable_free_page_optimization(void);
+void precopy_enable_metadata_migration(void);
void ram_mig_init(void);
void qemu_guest_free_page_hint(void *addr, size_t len);
diff --git a/migration/ram.c b/migration/ram.c
index 7811cde643..32f764680d 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -53,9 +53,11 @@
#include "block.h"
#include "sysemu/sysemu.h"
#include "sysemu/cpu-throttle.h"
+#include "sysemu/kvm.h"
#include "savevm.h"
#include "qemu/iov.h"
#include "multifd.h"
+#include "kvm_arm.h"
/***/
/* ram save/restore */
@@ -75,6 +77,9 @@
#define RAM_SAVE_FLAG_XBZRLE 0x40
/* 0x80 is reserved in migration.h start with 0x100 next */
#define RAM_SAVE_FLAG_COMPRESS_PAGE0x100
+#define RAM_SAVE_FLAG_MTE 0x200
+
+#define MTE_GRANULE_SIZE (16)
static inline bool is_zero_range(uint8_t *p, uint64_t size)
{
@@ -310,6 +315,8 @@ struct RAMState {
bool ram_bulk_stage;
/* The free page optimization is enabled */
bool fpo_enabled;
+/* The RAM meta data(e.t memory tag) is enabled */
+bool metadata_enabled;
/* How many times we have dirty too many pages */
int dirty_rate_high_cnt;
/* these variables are used for bitmap sync */
@@ -387,6 +394,15 @@ void precopy_enable_free_page_optimization(void)
ram_state->fpo_enabled = true;
}
+void precopy_enable_metadata_migration(void)
+{
+if (!ram_state) {
+return;
+}
+
+ram_state->metadata_enabled = true;
+}
+
uint64_t ram_bytes_remaining(void)
{
return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
@@ -1127,6 +1143,61 @@ static bool control_save_page(RAMState *rs, RAMBlock
*block, ram_addr_t offset,
return true;
}
+static int save_normal_page_mte_tags(QEMUFile *f, uint8_t *addr)
+{
+uint8_t *tag_buf = NULL;
+uint64_t ipa;
+int size = TARGET_PAGE_SIZE / MTE_GRANULE_SIZE;
+
+if (kvm_physical_memory_addr_from_host(kvm_state, addr, )) {
+/* Buffer for the page tags(one byte per tag) */
+tag_buf = g_try_malloc0(size);
+if (!tag_buf) {
+error_report("%s: Error allocating MTE tag_buf", __func__);
+return 0;
+}
+
+if (kvm_arm_mte_get_tags(ipa, TARGET_PAGE_SIZE, tag_buf) < 0) {
+error_report("%s: Can't get MTE tags from guest", __func__);
+g_free(tag_buf);
+return 0;
+}
+
+qemu_put_buffer(f, tag_buf, size);
+
+g_free(tag_buf);
+
+return size;
+}
+
+return 0;
+}
+
+static void load_normal_page_mte_tags(QEMUFile *f, uint8_t *addr)
+{
+uint8_t *tag_buf = NULL;
+uint64_t ipa;
+int size = TARGET_PAGE_SIZE / MTE_GRANULE_SIZE;
+
+if (kvm_physical_memory_addr_from_host(kvm_state, addr, )) {
+/* Buffer for the page tags(one byte per tag) */
+tag_buf = g_try_malloc0(size);
+if (!tag_buf) {
+error_report("%s: Error allocating MTE tag_buf", __func__);
+return;
+}
+
+qemu_get_buffer(f, tag_buf, size);
+if (kvm_arm_mte_set_tags(ipa, TARGET_PAGE_SIZE, tag_buf) < 0) {
+error_report("%s: Can't set MTE tags to guest", __func__);
+}
+
+g_free(tag_buf);
+}
+
+return;
+}
+
/*
* directly send the page to the stream
*
@@ -1141,6 +1212,10 @@ static bool control_save_page(RAMState *rs, RAMBlock
*block, ram_addr_t offset,
static int save_normal_page(RAMState *rs, RAMBlock *block, ram_addr_t offset,
uint8_t *buf, bool async)
Re: [PATCH v5 0/2] MTE support for KVM guest
On Wed, 16 Dec 2020 at 18:23, Steven Price wrote: > > On 16/12/2020 07:31, Haibo Xu wrote: > [...] > > Hi Steve, > > Hi Haibo > > > I have finished verifying the POC on a FVP setup, and the MTE test case can > > be migrated from one VM to another successfully. Since the test case is very > > simple which just maps one page with MTE enabled and does some memory > > access, so I can't say it's OK for other cases. > > That's great progress. > > > > > BTW, I noticed that you have sent out patch set v6 which mentions that > > mapping > > all the guest memory with PROT_MTE was not feasible. So what's the plan for > > the > > next step? Will new KVM APIs which can facilitate the tag store and recover > > be > > available? > > I'm currently rebasing on top of the KASAN MTE patch series. My plan for > now is to switch back to not requiring the VMM to supply PROT_MTE (so > KVM 'upgrades' the pages as necessary) and I'll add an RFC patch on the > end of the series to add an KVM API for doing bulk read/write of tags. > That way the VMM can map guest memory without PROT_MTE (so device 'DMA' > accesses will be unchecked), and use the new API for migration. > Great! Will have a try with the new API in my POC! > Thanks, > > Steve
Re: [PATCH v5 0/2] MTE support for KVM guest
On Mon, 7 Dec 2020 at 22:48, Steven Price wrote: > > On 04/12/2020 08:25, Haibo Xu wrote: > > On Fri, 20 Nov 2020 at 17:51, Steven Price wrote: > >> > >> On 19/11/2020 19:11, Marc Zyngier wrote: > >>> On 2020-11-19 18:42, Andrew Jones wrote: > >>>> On Thu, Nov 19, 2020 at 03:45:40PM +, Peter Maydell wrote: > >>>>> On Thu, 19 Nov 2020 at 15:39, Steven Price wrote: > >>>>>> This series adds support for Arm's Memory Tagging Extension (MTE) to > >>>>>> KVM, allowing KVM guests to make use of it. This builds on the > >>>>> existing > >>>>>> user space support already in v5.10-rc1, see [1] for an overview. > >>>>> > >>>>>> The change to require the VMM to map all guest memory PROT_MTE is > >>>>>> significant as it means that the VMM has to deal with the MTE tags > >>>>> even > >>>>>> if it doesn't care about them (e.g. for virtual devices or if the VMM > >>>>>> doesn't support migration). Also unfortunately because the VMM can > >>>>>> change the memory layout at any time the check for PROT_MTE/VM_MTE has > >>>>>> to be done very late (at the point of faulting pages into stage 2). > >>>>> > >>>>> I'm a bit dubious about requring the VMM to map the guest memory > >>>>> PROT_MTE unless somebody's done at least a sketch of the design > >>>>> for how this would work on the QEMU side. Currently QEMU just > >>>>> assumes the guest memory is guest memory and it can access it > >>>>> without special precautions... > >>>>> > >>>> > >>>> There are two statements being made here: > >>>> > >>>> 1) Requiring the use of PROT_MTE when mapping guest memory may not fit > >>>> QEMU well. > >>>> > >>>> 2) New KVM features should be accompanied with supporting QEMU code in > >>>> order to prove that the APIs make sense. > >>>> > >>>> I strongly agree with (2). While kvmtool supports some quick testing, it > >>>> doesn't support migration. We must test all new features with a migration > >>>> supporting VMM. > >>>> > >>>> I'm not sure about (1). I don't feel like it should be a major problem, > >>>> but (2). > >> > >> (1) seems to be contentious whichever way we go. Either PROT_MTE isn't > >> required in which case it's easy to accidentally screw up migration, or > >> it is required in which case it's difficult to handle normal guest > >> memory from the VMM. I get the impression that probably I should go back > >> to the previous approach - sorry for the distraction with this change. > >> > >> (2) isn't something I'm trying to skip, but I'm limited in what I can do > >> myself so would appreciate help here. Haibo is looking into this. > >> > > > > Hi Steven, > > > > Sorry for the later reply! > > > > I have finished the POC for the MTE migration support with the assumption > > that all the memory is mapped with PROT_MTE. But I got stuck in the test > > with a FVP setup. Previously, I successfully compiled a test case to verify > > the basic function of MTE in a guest. But these days, the re-compiled test > > can't be executed by the guest(very weird). The short plan to verify > > the migration > > is to set the MTE tags on one page in the guest, and try to dump the > > migrated > > memory contents. > > Hi Haibo, > > Sounds like you are making good progress - thanks for the update. Have > you thought about how the PROT_MTE mappings might work if QEMU itself > were to use MTE? My worry is that we end up with MTE in a guest > preventing QEMU from using MTE itself (because of the PROT_MTE > mappings). I'm hoping QEMU can wrap its use of guest memory in a > sequence which disables tag checking (something similar will be needed > for the "protected VM" use case anyway), but this isn't something I've > looked into. > > > I will update the status later next week! > > Great, I look forward to hearing how it goes. Hi Steve, I have finished verifying the POC on a FVP setup, and the MTE test case can be migrated from one VM to another successfully. Since the test case is very simple which just maps one page with MTE enabled and does some memory access, so I can't say it's OK for other cases. BTW, I noticed that you have sent out patch set v6 which mentions that mapping all the guest memory with PROT_MTE was not feasible. So what's the plan for the next step? Will new KVM APIs which can facilitate the tag store and recover be available? Regards, Haibo > > Thanks, > > Steve
Re: [PATCH v5 0/2] MTE support for KVM guest
On Tue, 8 Dec 2020 at 18:01, Marc Zyngier wrote: > > On 2020-12-08 09:51, Haibo Xu wrote: > > On Mon, 7 Dec 2020 at 22:48, Steven Price wrote: > >> > > [...] > > >> Sounds like you are making good progress - thanks for the update. Have > >> you thought about how the PROT_MTE mappings might work if QEMU itself > >> were to use MTE? My worry is that we end up with MTE in a guest > >> preventing QEMU from using MTE itself (because of the PROT_MTE > >> mappings). I'm hoping QEMU can wrap its use of guest memory in a > >> sequence which disables tag checking (something similar will be needed > >> for the "protected VM" use case anyway), but this isn't something I've > >> looked into. > > > > As far as I can see, to map all the guest memory with PROT_MTE in VMM > > is a little weird, and lots of APIs have to be changed to include this > > flag. > > IMHO, it would be better if the KVM can provide new APIs to load/store > > the > > guest memory tag which may make it easier to enable the Qemu migration > > support. > > On what granularity? To what storage? How do you plan to synchronise > this > with the dirty-log interface? The Qemu would migrate page by page, and if one page has been migrated but becomes dirty again, the migration process would re-send this dirty page. The current MTE migration POC codes would try to send the page tags just after the page data, if one page becomes dirty again, the page data and the tags would be re-sent. > > Thanks, > > M. > -- > Jazz is not dead. It just smells funny...
Re: [PATCH v5 0/2] MTE support for KVM guest
On Tue, 8 Dec 2020 at 00:44, Dr. David Alan Gilbert wrote: > > * Steven Price (steven.pr...@arm.com) wrote: > > On 07/12/2020 15:27, Peter Maydell wrote: > > > On Mon, 7 Dec 2020 at 14:48, Steven Price wrote: > > > > Sounds like you are making good progress - thanks for the update. Have > > > > you thought about how the PROT_MTE mappings might work if QEMU itself > > > > were to use MTE? My worry is that we end up with MTE in a guest > > > > preventing QEMU from using MTE itself (because of the PROT_MTE > > > > mappings). I'm hoping QEMU can wrap its use of guest memory in a > > > > sequence which disables tag checking (something similar will be needed > > > > for the "protected VM" use case anyway), but this isn't something I've > > > > looked into. > > > > > > It's not entirely the same as the "protected VM" case. For that > > > the patches currently on list basically special case "this is a > > > debug access (eg from gdbstub/monitor)" which then either gets > > > to go via "decrypt guest RAM for debug" or gets failed depending > > > on whether the VM has a debug-is-ok flag enabled. For an MTE > > > guest the common case will be guests doing standard DMA operations > > > to or from guest memory. The ideal API for that from QEMU's > > > point of view would be "accesses to guest RAM don't do tag > > > checks, even if tag checks are enabled for accesses QEMU does to > > > memory it has allocated itself as a normal userspace program". > > > > Sorry, I know I simplified it rather by saying it's similar to protected VM. > > Basically as I see it there are three types of memory access: > > > > 1) Debug case - has to go via a special case for decryption or ignoring the > > MTE tag value. Hopefully this can be abstracted in the same way. > > > > 2) Migration - for a protected VM there's likely to be a special method to > > allow the VMM access to the encrypted memory (AFAIK memory is usually kept > > inaccessible to the VMM). For MTE this again has to be special cased as we > > actually want both the data and the tag values. > > > > 3) Device DMA - for a protected VM it's usual to unencrypt a small area of > > memory (with the permission of the guest) and use that as a bounce buffer. > > This is possible with MTE: have an area the VMM purposefully maps with > > PROT_MTE. The issue is that this has a performance overhead and we can do > > better with MTE because it's trivial for the VMM to disable the protection > > for any memory. > > Those all sound very similar to the AMD SEV world; there's the special > case for Debug that Peter mentioned; migration is ...complicated and > needs special case that's still being figured out, and as I understand > Device DMA also uses a bounce buffer (and swiotlb in the guest to make > that happen). > > > I'm not sure about the stories for the IBM hardware equivalents. Like s390-skeys(storage keys) support in Qemu? I have read the migration support for the s390-skeys in Qemu and found that the logic is very similar to that of MTE, except the difference that the s390-skeys were migrated separately from that of the guest memory data while for MTE, I think the guest memory tags should go with the memory data. > > Dave > > > The part I'm unsure on is how easy it is for QEMU to deal with (3) without > > the overhead of bounce buffers. Ideally there'd already be a wrapper for > > guest memory accesses and that could just be wrapped with setting TCO during > > the access. I suspect the actual situation is more complex though, and I'm > > hoping Haibo's investigations will help us understand this. > > > > Thanks, > > > > Steve > > > -- > Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK >
Re: [PATCH v5 0/2] MTE support for KVM guest
On Mon, 7 Dec 2020 at 22:48, Steven Price wrote: > > On 04/12/2020 08:25, Haibo Xu wrote: > > On Fri, 20 Nov 2020 at 17:51, Steven Price wrote: > >> > >> On 19/11/2020 19:11, Marc Zyngier wrote: > >>> On 2020-11-19 18:42, Andrew Jones wrote: > >>>> On Thu, Nov 19, 2020 at 03:45:40PM +, Peter Maydell wrote: > >>>>> On Thu, 19 Nov 2020 at 15:39, Steven Price wrote: > >>>>>> This series adds support for Arm's Memory Tagging Extension (MTE) to > >>>>>> KVM, allowing KVM guests to make use of it. This builds on the > >>>>> existing > >>>>>> user space support already in v5.10-rc1, see [1] for an overview. > >>>>> > >>>>>> The change to require the VMM to map all guest memory PROT_MTE is > >>>>>> significant as it means that the VMM has to deal with the MTE tags > >>>>> even > >>>>>> if it doesn't care about them (e.g. for virtual devices or if the VMM > >>>>>> doesn't support migration). Also unfortunately because the VMM can > >>>>>> change the memory layout at any time the check for PROT_MTE/VM_MTE has > >>>>>> to be done very late (at the point of faulting pages into stage 2). > >>>>> > >>>>> I'm a bit dubious about requring the VMM to map the guest memory > >>>>> PROT_MTE unless somebody's done at least a sketch of the design > >>>>> for how this would work on the QEMU side. Currently QEMU just > >>>>> assumes the guest memory is guest memory and it can access it > >>>>> without special precautions... > >>>>> > >>>> > >>>> There are two statements being made here: > >>>> > >>>> 1) Requiring the use of PROT_MTE when mapping guest memory may not fit > >>>> QEMU well. > >>>> > >>>> 2) New KVM features should be accompanied with supporting QEMU code in > >>>> order to prove that the APIs make sense. > >>>> > >>>> I strongly agree with (2). While kvmtool supports some quick testing, it > >>>> doesn't support migration. We must test all new features with a migration > >>>> supporting VMM. > >>>> > >>>> I'm not sure about (1). I don't feel like it should be a major problem, > >>>> but (2). > >> > >> (1) seems to be contentious whichever way we go. Either PROT_MTE isn't > >> required in which case it's easy to accidentally screw up migration, or > >> it is required in which case it's difficult to handle normal guest > >> memory from the VMM. I get the impression that probably I should go back > >> to the previous approach - sorry for the distraction with this change. > >> > >> (2) isn't something I'm trying to skip, but I'm limited in what I can do > >> myself so would appreciate help here. Haibo is looking into this. > >> > > > > Hi Steven, > > > > Sorry for the later reply! > > > > I have finished the POC for the MTE migration support with the assumption > > that all the memory is mapped with PROT_MTE. But I got stuck in the test > > with a FVP setup. Previously, I successfully compiled a test case to verify > > the basic function of MTE in a guest. But these days, the re-compiled test > > can't be executed by the guest(very weird). The short plan to verify > > the migration > > is to set the MTE tags on one page in the guest, and try to dump the > > migrated > > memory contents. > > Hi Haibo, > > Sounds like you are making good progress - thanks for the update. Have > you thought about how the PROT_MTE mappings might work if QEMU itself > were to use MTE? My worry is that we end up with MTE in a guest > preventing QEMU from using MTE itself (because of the PROT_MTE > mappings). I'm hoping QEMU can wrap its use of guest memory in a > sequence which disables tag checking (something similar will be needed > for the "protected VM" use case anyway), but this isn't something I've > looked into. As far as I can see, to map all the guest memory with PROT_MTE in VMM is a little weird, and lots of APIs have to be changed to include this flag. IMHO, it would be better if the KVM can provide new APIs to load/store the guest memory tag which may make it easier to enable the Qemu migration support. > > > I will update the status later next week! > > Great, I look forward to hearing how it goes. > > Thanks, > > Steve
Re: [PATCH v5 0/2] MTE support for KVM guest
On Fri, 20 Nov 2020 at 17:51, Steven Price wrote: > > On 19/11/2020 19:11, Marc Zyngier wrote: > > On 2020-11-19 18:42, Andrew Jones wrote: > >> On Thu, Nov 19, 2020 at 03:45:40PM +, Peter Maydell wrote: > >>> On Thu, 19 Nov 2020 at 15:39, Steven Price wrote: > >>> > This series adds support for Arm's Memory Tagging Extension (MTE) to > >>> > KVM, allowing KVM guests to make use of it. This builds on the > >>> existing > >>> > user space support already in v5.10-rc1, see [1] for an overview. > >>> > >>> > The change to require the VMM to map all guest memory PROT_MTE is > >>> > significant as it means that the VMM has to deal with the MTE tags > >>> even > >>> > if it doesn't care about them (e.g. for virtual devices or if the VMM > >>> > doesn't support migration). Also unfortunately because the VMM can > >>> > change the memory layout at any time the check for PROT_MTE/VM_MTE has > >>> > to be done very late (at the point of faulting pages into stage 2). > >>> > >>> I'm a bit dubious about requring the VMM to map the guest memory > >>> PROT_MTE unless somebody's done at least a sketch of the design > >>> for how this would work on the QEMU side. Currently QEMU just > >>> assumes the guest memory is guest memory and it can access it > >>> without special precautions... > >>> > >> > >> There are two statements being made here: > >> > >> 1) Requiring the use of PROT_MTE when mapping guest memory may not fit > >>QEMU well. > >> > >> 2) New KVM features should be accompanied with supporting QEMU code in > >>order to prove that the APIs make sense. > >> > >> I strongly agree with (2). While kvmtool supports some quick testing, it > >> doesn't support migration. We must test all new features with a migration > >> supporting VMM. > >> > >> I'm not sure about (1). I don't feel like it should be a major problem, > >> but (2). > > (1) seems to be contentious whichever way we go. Either PROT_MTE isn't > required in which case it's easy to accidentally screw up migration, or > it is required in which case it's difficult to handle normal guest > memory from the VMM. I get the impression that probably I should go back > to the previous approach - sorry for the distraction with this change. > > (2) isn't something I'm trying to skip, but I'm limited in what I can do > myself so would appreciate help here. Haibo is looking into this. > Hi Steven, Sorry for the later reply! I have finished the POC for the MTE migration support with the assumption that all the memory is mapped with PROT_MTE. But I got stuck in the test with a FVP setup. Previously, I successfully compiled a test case to verify the basic function of MTE in a guest. But these days, the re-compiled test can't be executed by the guest(very weird). The short plan to verify the migration is to set the MTE tags on one page in the guest, and try to dump the migrated memory contents. I will update the status later next week! Regards, Haibo > >> > >> I'd be happy to help with the QEMU prototype, but preferably when there's > >> hardware available. Has all the current MTE testing just been done on > >> simulators? And, if so, are there regression tests regularly running on > >> the simulators too? And can they test migration? If hardware doesn't > >> show up quickly and simulators aren't used for regression tests, then > >> all this code will start rotting from day one. > > As Marc says, hardware isn't available. Testing is either via the Arm > FVP model (that I've been using for most of my testing) or QEMU full > system emulation. > > > > > While I agree with the sentiment, the reality is pretty bleak. > > > > I'm pretty sure nobody will ever run a migration on emulation. I also doubt > > there is much overlap between MTE users and migration users, unfortunately. > > > > No HW is available today, and when it becomes available, it will be in > > the form of a closed system on which QEMU doesn't run, either because > > we are locked out of EL2 (as usual), or because migration is not part of > > the use case (like KVM on Android, for example). > > > > So we can wait another two (five?) years until general purpose HW becomes > > available, or we start merging what we can test today. I'm inclined to > > do the latter. > > > > And I think it is absolutely fine for QEMU to say "no MTE support with KVM" > > (we can remove all userspace visibility, except for the capability). > > What I'm trying to achieve is a situation where KVM+MTE without > migration works and we leave ourselves a clear path where migration can > be added. With hindsight I think this version of the series was a wrong > turn - if we return to not requiring PROT_MTE then we have the following > two potential options to explore for migration in the future: > > * The VMM can choose to enable PROT_MTE if it needs to, and if desired > we can add a flag to enforce this in the kernel. > > * If needed a new kernel interface can be provided to fetch/set tags > from guest memory which isn't
Re: [PATCH v3 07/12] hw/arm/virt: Move post cpu realize check into its own function
On Tue, 15 Sep 2020 at 15:31, Andrew Jones wrote:
>
> On Tue, Sep 15, 2020 at 03:03:49PM +0800, Haibo Xu wrote:
> > On Tue, 15 Sep 2020 at 14:22, Andrew Jones wrote:
> > >
> > > On Tue, Sep 15, 2020 at 03:11:43AM +, Haibo Xu wrote:
> > > > From: Andrew Jones
> > > >
> > > > We'll add more to this new function in coming patches so we also
> > > > state the gic must be created and call it below create_gic().
> > > >
> > > > No functional change intended.
> > > >
> > > > Signed-off-by: Andrew Jones
> > > > Reviewed-by: Peter Maydell
> > > > ---
> > > > hw/arm/virt.c | 38 ++
> > > > 1 file changed, 22 insertions(+), 16 deletions(-)
> > >
> > > This still isn't the right version of this patch. You need
> > > https://www.mail-archive.com/qemu-devel@nongnu.org/msg727591.html
> > >
> >
> > My fault! Very sorry for forgetting to address that in this version.
> > Shall I re-send this patch with the fix? Or fix it in the next version?
>
> I can wait. It'll be awhile before this series can be merged anyway, since
> the KVM patches haven't been merged yet.
>
> Thanks,
> drew
>
Ok, I will fix it in the next version.
Thanks again for the review!
Regards,
Haibo
> >
> > > >
> > > > diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> > > > index 3f6d26c531..2ffcb073af 100644
> > > > --- a/hw/arm/virt.c
> > > > +++ b/hw/arm/virt.c
> > > > @@ -1672,6 +1672,26 @@ static void
> > > > finalize_gic_version(VirtMachineState *vms)
> > > > }
> > > > }
> > > >
> > > > +static void virt_cpu_post_init(VirtMachineState *vms)
> > > > +{
> > > > +bool aarch64;
> > > > +
> > > > +aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64",
> > > > NULL);
> > > > +
> > > > +if (!kvm_enabled()) {
> > > > +if (aarch64 && vms->highmem) {
> > > > +int requested_pa_size = 64 - clz64(vms->highest_gpa);
> > > > +int pamax = arm_pamax(ARM_CPU(first_cpu));
> > > > +
> > > > +if (pamax < requested_pa_size) {
> > > > +error_report("VCPU supports less PA bits (%d) than
> > > > requested "
> > > > +"by the memory map (%d)", pamax,
> > > > requested_pa_size);
> > > > +exit(1);
> > > > +}
> > > > +}
> > > > + }
> > > > +}
> > > > +
> > > > static void machvirt_init(MachineState *machine)
> > > > {
> > > > VirtMachineState *vms = VIRT_MACHINE(machine);
> > > > @@ -1890,22 +1910,6 @@ static void machvirt_init(MachineState *machine)
> > > > fdt_add_timer_nodes(vms);
> > > > fdt_add_cpu_nodes(vms);
> > > >
> > > > - if (!kvm_enabled()) {
> > > > -ARMCPU *cpu = ARM_CPU(first_cpu);
> > > > -bool aarch64 = object_property_get_bool(OBJECT(cpu),
> > > > "aarch64", NULL);
> > > > -
> > > > -if (aarch64 && vms->highmem) {
> > > > -int requested_pa_size, pamax = arm_pamax(cpu);
> > > > -
> > > > -requested_pa_size = 64 - clz64(vms->highest_gpa);
> > > > -if (pamax < requested_pa_size) {
> > > > -error_report("VCPU supports less PA bits (%d) than
> > > > requested "
> > > > -"by the memory map (%d)", pamax,
> > > > requested_pa_size);
> > > > -exit(1);
> > > > -}
> > > > -}
> > > > -}
> > > > -
> > > > memory_region_add_subregion(sysmem, vms->memmap[VIRT_MEM].base,
> > > > machine->ram);
> > > > if (machine->device_memory) {
> > > > @@ -1917,6 +1921,8 @@ static void machvirt_init(MachineState *machine)
> > > >
> > > > create_gic(vms);
> > > >
> > > > +virt_cpu_post_init(vms);
> > > > +
> > > > fdt_add_pmu_nodes(vms);
> > > >
> > > > create_uart(vms, VIRT_UART, sysmem, serial_hd(0));
> > > > --
> > > > 2.17.1
> > > >
> > > >
> > >
> >
>
Re: [PATCH v3 03/12] target/arm/cpu: spe: Add an option to turn on/off vSPE support
On Tue, 15 Sep 2020 at 14:11, Andrew Jones wrote:
>
> On Tue, Sep 15, 2020 at 03:11:39AM +, Haibo Xu wrote:
> > Adds a spe=[on/off] option to enable/disable vSPE support in
> > guest vCPU.
> >
> > Reviewed-by: Andrew Jones
> > Signed-off-by: Haibo Xu
> > ---
> > target/arm/cpu.c | 6 ++
> > target/arm/cpu.h | 13
> > target/arm/cpu64.c | 52 ++
> > 3 files changed, 71 insertions(+)
> >
> > diff --git a/target/arm/cpu.c b/target/arm/cpu.c
> > index c179e0752d..f211958eaa 100644
> > --- a/target/arm/cpu.c
> > +++ b/target/arm/cpu.c
> > @@ -1310,6 +1310,12 @@ void arm_cpu_finalize_features(ARMCPU *cpu, Error
> > **errp)
> > error_propagate(errp, local_err);
> > return;
> > }
> > +
> > +arm_cpu_spe_finalize(cpu, _err);
> > +if (local_err != NULL) {
> > +error_propagate(errp, local_err);
> > +return;
> > +}
> > }
> > }
> >
> > diff --git a/target/arm/cpu.h b/target/arm/cpu.h
> > index a1c7d8ebae..baf2bbcee8 100644
> > --- a/target/arm/cpu.h
> > +++ b/target/arm/cpu.h
> > @@ -24,6 +24,7 @@
> > #include "hw/registerfields.h"
> > #include "cpu-qom.h"
> > #include "exec/cpu-defs.h"
> > +#include "qapi/qapi-types-common.h"
> >
> > /* ARM processors have a weak memory model */
> > #define TCG_GUEST_DEFAULT_MO (0)
> > @@ -196,9 +197,11 @@ typedef struct {
> > #ifdef TARGET_AARCH64
> > # define ARM_MAX_VQ16
> > void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp);
> > +void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp);
> > #else
> > # define ARM_MAX_VQ1
> > static inline void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) { }
> > +static inline void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp) { }
> > #endif
> >
> > typedef struct ARMVectorReg {
> > @@ -829,6 +832,8 @@ struct ARMCPU {
> > bool has_el3;
> > /* CPU has PMU (Performance Monitor Unit) */
> > bool has_pmu;
> > +/* CPU has SPE (Statistical Profiling Extension) */
> > +OnOffAuto has_spe;
> > /* CPU has VFP */
> > bool has_vfp;
> > /* CPU has Neon */
> > @@ -3869,6 +3874,14 @@ static inline bool isar_feature_aa64_pmu_8_4(const
> > ARMISARegisters *id)
> > FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) != 0xf;
> > }
> >
> > +/*
> > + * Currently we don't differentiate between the ARMv8.2-SPE and
> > ARMv8.3-SPE.
> > + */
> > +static inline bool isar_feature_aa64_spe(const ARMISARegisters *id)
> > +{
> > +return FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMSVER) != 0;
> > +}
> > +
> > static inline bool isar_feature_aa64_rcpc_8_3(const ARMISARegisters *id)
> > {
> > return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, LRCPC) != 0;
> > diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
> > index 3c2b3d9599..4997c4a3c0 100644
> > --- a/target/arm/cpu64.c
> > +++ b/target/arm/cpu64.c
> > @@ -572,6 +572,55 @@ void aarch64_add_sve_properties(Object *obj)
> > }
> > }
> >
> > +void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp)
> > +{
> > +uint64_t t;
> > +uint32_t value = 0;
> > +
> > +if (cpu->has_spe == ON_OFF_AUTO_AUTO) {
> > +if (kvm_enabled() && kvm_arm_spe_supported()) {
> > +cpu->has_spe = ON_OFF_AUTO_ON;
> > +} else {
> > +cpu->has_spe = ON_OFF_AUTO_OFF;
> > +}
> > +} else if (cpu->has_spe == ON_OFF_AUTO_ON) {
> > +if (!kvm_enabled() || !kvm_arm_spe_supported()) {
> > +error_setg(errp, "'spe' cannot be enabled on this host");
> > +return;
> > +}
> > +}
> > +
> > +/*
> > + * According to the ARM ARM, the ID_AA64DFR0[PMSVER] currently
> > + * support 3 values:
> > + *
> > + * 0b: SPE not implemented
> > + * 0b0001: ARMv8.2-SPE implemented
> > + * 0b0010: ARMv8.3-SPE implemented
> > + *
> > + * But the kernel KVM API didn't expose all these 3 values, and
> > + * we can only get whether the SPE feature is supported or not.
> > + * So here we just set the PMSVER to 1 if this feature was supported.
> > + */
> > +if (cpu->has_spe == ON_OFF_AUTO_ON) {
Re: [PATCH v3 07/12] hw/arm/virt: Move post cpu realize check into its own function
On Tue, 15 Sep 2020 at 14:22, Andrew Jones wrote:
>
> On Tue, Sep 15, 2020 at 03:11:43AM +, Haibo Xu wrote:
> > From: Andrew Jones
> >
> > We'll add more to this new function in coming patches so we also
> > state the gic must be created and call it below create_gic().
> >
> > No functional change intended.
> >
> > Signed-off-by: Andrew Jones
> > Reviewed-by: Peter Maydell
> > ---
> > hw/arm/virt.c | 38 ++
> > 1 file changed, 22 insertions(+), 16 deletions(-)
>
> This still isn't the right version of this patch. You need
> https://www.mail-archive.com/qemu-devel@nongnu.org/msg727591.html
>
My fault! Very sorry for forgetting to address that in this version.
Shall I re-send this patch with the fix? Or fix it in the next version?
> >
> > diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> > index 3f6d26c531..2ffcb073af 100644
> > --- a/hw/arm/virt.c
> > +++ b/hw/arm/virt.c
> > @@ -1672,6 +1672,26 @@ static void finalize_gic_version(VirtMachineState
> > *vms)
> > }
> > }
> >
> > +static void virt_cpu_post_init(VirtMachineState *vms)
> > +{
> > +bool aarch64;
> > +
> > +aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL);
> > +
> > +if (!kvm_enabled()) {
> > +if (aarch64 && vms->highmem) {
> > +int requested_pa_size = 64 - clz64(vms->highest_gpa);
> > +int pamax = arm_pamax(ARM_CPU(first_cpu));
> > +
> > +if (pamax < requested_pa_size) {
> > +error_report("VCPU supports less PA bits (%d) than
> > requested "
> > +"by the memory map (%d)", pamax,
> > requested_pa_size);
> > +exit(1);
> > +}
> > +}
> > + }
> > +}
> > +
> > static void machvirt_init(MachineState *machine)
> > {
> > VirtMachineState *vms = VIRT_MACHINE(machine);
> > @@ -1890,22 +1910,6 @@ static void machvirt_init(MachineState *machine)
> > fdt_add_timer_nodes(vms);
> > fdt_add_cpu_nodes(vms);
> >
> > - if (!kvm_enabled()) {
> > -ARMCPU *cpu = ARM_CPU(first_cpu);
> > -bool aarch64 = object_property_get_bool(OBJECT(cpu), "aarch64",
> > NULL);
> > -
> > -if (aarch64 && vms->highmem) {
> > -int requested_pa_size, pamax = arm_pamax(cpu);
> > -
> > -requested_pa_size = 64 - clz64(vms->highest_gpa);
> > -if (pamax < requested_pa_size) {
> > -error_report("VCPU supports less PA bits (%d) than
> > requested "
> > -"by the memory map (%d)", pamax,
> > requested_pa_size);
> > -exit(1);
> > -}
> > -}
> > -}
> > -
> > memory_region_add_subregion(sysmem, vms->memmap[VIRT_MEM].base,
> > machine->ram);
> > if (machine->device_memory) {
> > @@ -1917,6 +1921,8 @@ static void machvirt_init(MachineState *machine)
> >
> > create_gic(vms);
> >
> > +virt_cpu_post_init(vms);
> > +
> > fdt_add_pmu_nodes(vms);
> >
> > create_uart(vms, VIRT_UART, sysmem, serial_hd(0));
> > --
> > 2.17.1
> >
> >
>
[PATCH v3 10/12] target/arm/cpu: spe: Enable spe to work with host cpu
Turn on the spe cpu property by default if host cpu
support it, i.e. we can now do '-cpu max|host' to add
the vSPE, and '-cpu max|host,spe=off' to remove it.
Signed-off-by: Haibo Xu
---
target/arm/cpu.c | 1 +
target/arm/cpu.h | 2 ++
target/arm/cpu64.c | 10 +++---
target/arm/kvm64.c | 12
4 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 786cc6134c..05a4b7d208 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -2271,6 +2271,7 @@ static void arm_host_initfn(Object *obj)
kvm_arm_set_cpu_features_from_host(cpu);
if (arm_feature(>env, ARM_FEATURE_AARCH64)) {
aarch64_add_sve_properties(obj);
+aarch64_add_spe_properties(obj);
}
arm_cpu_post_init(obj);
}
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 395a1e5df8..5a3ea876c8 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -1040,6 +1040,7 @@ void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq);
void aarch64_sve_change_el(CPUARMState *env, int old_el,
int new_el, bool el0_a64);
void aarch64_add_sve_properties(Object *obj);
+void aarch64_add_spe_properties(Object *obj);
/*
* SVE registers are encoded in KVM's memory in an endianness-invariant format.
@@ -1071,6 +1072,7 @@ static inline void aarch64_sve_change_el(CPUARMState
*env, int o,
int n, bool a)
{ }
static inline void aarch64_add_sve_properties(Object *obj) { }
+static inline void aarch64_add_spe_properties(Object *obj) { }
#endif
#if !defined(CONFIG_TCG)
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 4997c4a3c0..04daeb901d 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -621,6 +621,12 @@ static void arm_spe_set(Object *obj, bool value, Error
**errp)
ARM_CPU(obj)->has_spe = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
}
+void aarch64_add_spe_properties(Object *obj)
+{
+ARM_CPU(obj)->has_spe = ON_OFF_AUTO_AUTO;
+object_property_add_bool(obj, "spe", arm_spe_get, arm_spe_set);
+}
+
/* -cpu max: if KVM is enabled, like -cpu host (best possible with this host);
* otherwise, a CPU with as many features enabled as our emulation supports.
* The version of '-cpu max' for qemu-system-arm is defined in cpu.c;
@@ -632,6 +638,7 @@ static void aarch64_max_initfn(Object *obj)
if (kvm_enabled()) {
kvm_arm_set_cpu_features_from_host(cpu);
+aarch64_add_spe_properties(obj);
} else {
uint64_t t;
uint32_t u;
@@ -770,9 +777,6 @@ static void aarch64_max_initfn(Object *obj)
aarch64_add_sve_properties(obj);
object_property_add(obj, "sve-max-vq", "uint32", cpu_max_get_sve_max_vq,
cpu_max_set_sve_max_vq, NULL, NULL);
-
-cpu->has_spe = ON_OFF_AUTO_AUTO;
-object_property_add_bool(obj, "spe", arm_spe_get, arm_spe_set);
}
static const ARMCPUInfo aarch64_cpus[] = {
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 5a2032fc9e..20269efeff 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -515,6 +515,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
*/
int fdarray[3];
bool sve_supported;
+bool spe_supported;
uint64_t features = 0;
uint64_t t;
int err;
@@ -655,6 +656,8 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
}
sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) >
0;
+spe_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION,
+ KVM_CAP_ARM_SPE_V1) > 0;
kvm_arm_destroy_scratch_host_vcpu(fdarray);
@@ -668,6 +671,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
*ahcf)
t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
ahcf->isar.id_aa64pfr0 = t;
}
+if (spe_supported) {
+t = ahcf->isar.id_aa64dfr0;
+t = FIELD_DP64(t, ID_AA64DFR0, PMSVER, 1);
+ahcf->isar.id_aa64dfr0 = t;
+}
/*
* We can assume any KVM supporting CPU is at least a v8
@@ -830,6 +838,10 @@ int kvm_arch_init_vcpu(CPUState *cs)
assert(kvm_arm_sve_supported());
cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
}
+if (cpu_isar_feature(aa64_spe, cpu)) {
+assert(kvm_arm_spe_supported());
+cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SPE_V1;
+}
/* Do KVM_ARM_VCPU_INIT ioctl */
ret = kvm_arm_vcpu_init(cs);
--
2.17.1
[PATCH v3 09/12] hw/arm/virt: spe: Add vSPE device and corresponding interrupt support
Add a virtual SPE device for virt machine while using
PPI 5 for SPE overflow interrupt number which has already
selected in kvmtool for the in-kernel irqchip support.
Reviewed-by: Andrew Jones
Signed-off-by: Haibo Xu
---
hw/arm/virt-acpi-build.c| 3 +++
hw/arm/virt.c | 43 -
include/hw/acpi/acpi-defs.h | 3 +++
include/hw/arm/virt.h | 1 +
target/arm/cpu.c| 2 ++
target/arm/cpu.h| 2 ++
6 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
index 9efd7a3881..3fd80fda53 100644
--- a/hw/arm/virt-acpi-build.c
+++ b/hw/arm/virt-acpi-build.c
@@ -665,6 +665,9 @@ build_madt(GArray *table_data, BIOSLinker *linker,
VirtMachineState *vms)
if (arm_feature(>env, ARM_FEATURE_PMU)) {
gicc->performance_interrupt = cpu_to_le32(PPI(VIRTUAL_PMU_IRQ));
}
+if (cpu_isar_feature(aa64_spe, armcpu)) {
+gicc->spe_interrupt = cpu_to_le32(PPI(VIRTUAL_SPE_IRQ));
+}
if (vms->virt) {
gicc->vgic_interrupt = cpu_to_le32(PPI(ARCH_GIC_MAINT_IRQ));
}
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 6bacfb668d..ee0831922b 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -545,6 +545,32 @@ static void fdt_add_pmu_nodes(const VirtMachineState *vms)
}
}
+static void fdt_add_spe_nodes(const VirtMachineState *vms)
+{
+ARMCPU *armcpu = ARM_CPU(first_cpu);
+uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
+
+if (!cpu_isar_feature(aa64_spe, armcpu)) {
+assert(!object_property_get_bool(OBJECT(armcpu), "spe", NULL));
+return;
+}
+
+if (vms->gic_version == VIRT_GIC_VERSION_2) {
+irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
+ GIC_FDT_IRQ_PPI_CPU_WIDTH,
+ (1 << vms->smp_cpus) - 1);
+}
+
+qemu_fdt_add_subnode(vms->fdt, "/spe");
+if (arm_feature(>env, ARM_FEATURE_V8)) {
+const char compat[] = "arm,statistical-profiling-extension-v1";
+qemu_fdt_setprop(vms->fdt, "/spe", "compatible",
+ compat, sizeof(compat));
+qemu_fdt_setprop_cells(vms->fdt, "/spe", "interrupts",
+ GIC_FDT_IRQ_TYPE_PPI, VIRTUAL_SPE_IRQ,
irqflags);
+}
+}
+
static inline DeviceState *create_acpi_ged(VirtMachineState *vms)
{
DeviceState *dev;
@@ -717,6 +743,10 @@ static void create_gic(VirtMachineState *vms)
qdev_get_gpio_in(vms->gic, ppibase
+ VIRTUAL_PMU_IRQ));
+qdev_connect_gpio_out_named(cpudev, "spe-interrupt", 0,
+qdev_get_gpio_in(vms->gic, ppibase
+ + VIRTUAL_SPE_IRQ));
+
sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev,
ARM_CPU_IRQ));
sysbus_connect_irq(gicbusdev, i + smp_cpus,
qdev_get_gpio_in(cpudev, ARM_CPU_FIQ));
@@ -1664,11 +1694,12 @@ static void finalize_gic_version(VirtMachineState *vms)
static void virt_cpu_post_init(VirtMachineState *vms)
{
-bool aarch64, pmu;
+bool aarch64, pmu, spe;
CPUState *cpu;
aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL);
pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL);
+spe = object_property_get_bool(OBJECT(first_cpu), "spe", NULL);
if (kvm_enabled()) {
CPU_FOREACH(cpu) {
@@ -1679,6 +1710,14 @@ static void virt_cpu_post_init(VirtMachineState *vms)
}
kvm_arm_pmu_init(cpu);
}
+
+if (spe) {
+assert(ARM_CPU(cpu)->has_spe == ON_OFF_AUTO_ON);
+if (kvm_irqchip_in_kernel()) {
+kvm_arm_spe_set_irq(cpu, PPI(VIRTUAL_SPE_IRQ));
+kvm_arm_spe_init(cpu);
+}
+}
}
} else {
if (aarch64 && vms->highmem) {
@@ -1927,6 +1966,8 @@ static void machvirt_init(MachineState *machine)
fdt_add_pmu_nodes(vms);
+fdt_add_spe_nodes(vms);
+
create_uart(vms, VIRT_UART, sysmem, serial_hd(0));
if (vms->secure) {
diff --git a/include/hw/acpi/acpi-defs.h b/include/hw/acpi/acpi-defs.h
index 38a42f409a..21e58f27c5 100644
--- a/include/hw/acpi/acpi-defs.h
+++ b/include/hw/acpi/acpi-defs.h
@@ -302,6 +302,9 @@ struct AcpiMadtGenericCpuInterface {
uint32_t vgic_interrupt;
uint64_t gicr_base_address;
uint64_t arm_mpidr;
+uint8_t efficiency_class;
+uint8_t reserved2[1];
+uint16_t spe_interrupt; /* ACPI 6.3 */
} QEMU_PACKED;
typedef struct AcpiMadtGenericCpuInterface AcpiMadtGenericCpuInterface;
diff
[PATCH v3 08/12] hw/arm/virt: Move kvm pmu setup to virt_cpu_post_init
From: Andrew Jones
Move the KVM PMU setup part of fdt_add_pmu_nodes() to
virt_cpu_post_init(), which is a more appropriate location. Now
fdt_add_pmu_nodes() is also named more appropriately, because it
no longer does anything but fdt node creation.
No functional change intended.
Signed-off-by: Andrew Jones
Reviewed-by: Peter Maydell
---
hw/arm/virt.c | 34 ++
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 2ffcb073af..6bacfb668d 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -521,21 +521,12 @@ static void fdt_add_gic_node(VirtMachineState *vms)
static void fdt_add_pmu_nodes(const VirtMachineState *vms)
{
-CPUState *cpu;
-ARMCPU *armcpu;
+ARMCPU *armcpu = ARM_CPU(first_cpu);
uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
-CPU_FOREACH(cpu) {
-armcpu = ARM_CPU(cpu);
-if (!arm_feature(>env, ARM_FEATURE_PMU)) {
-return;
-}
-if (kvm_enabled()) {
-if (kvm_irqchip_in_kernel()) {
-kvm_arm_pmu_set_irq(cpu, PPI(VIRTUAL_PMU_IRQ));
-}
-kvm_arm_pmu_init(cpu);
-}
+if (!arm_feature(>env, ARM_FEATURE_PMU)) {
+assert(!object_property_get_bool(OBJECT(armcpu), "pmu", NULL));
+return;
}
if (vms->gic_version == VIRT_GIC_VERSION_2) {
@@ -544,7 +535,6 @@ static void fdt_add_pmu_nodes(const VirtMachineState *vms)
(1 << vms->smp_cpus) - 1);
}
-armcpu = ARM_CPU(qemu_get_cpu(0));
qemu_fdt_add_subnode(vms->fdt, "/pmu");
if (arm_feature(>env, ARM_FEATURE_V8)) {
const char compat[] = "arm,armv8-pmuv3";
@@ -1674,11 +1664,23 @@ static void finalize_gic_version(VirtMachineState *vms)
static void virt_cpu_post_init(VirtMachineState *vms)
{
-bool aarch64;
+bool aarch64, pmu;
+CPUState *cpu;
aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL);
+pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL);
-if (!kvm_enabled()) {
+if (kvm_enabled()) {
+CPU_FOREACH(cpu) {
+if (pmu) {
+assert(arm_feature(_CPU(cpu)->env, ARM_FEATURE_PMU));
+if (kvm_irqchip_in_kernel()) {
+kvm_arm_pmu_set_irq(cpu, PPI(VIRTUAL_PMU_IRQ));
+}
+kvm_arm_pmu_init(cpu);
+}
+}
+} else {
if (aarch64 && vms->highmem) {
int requested_pa_size = 64 - clz64(vms->highest_gpa);
int pamax = arm_pamax(ARM_CPU(first_cpu));
--
2.17.1
[PATCH v3 07/12] hw/arm/virt: Move post cpu realize check into its own function
From: Andrew Jones
We'll add more to this new function in coming patches so we also
state the gic must be created and call it below create_gic().
No functional change intended.
Signed-off-by: Andrew Jones
Reviewed-by: Peter Maydell
---
hw/arm/virt.c | 38 ++
1 file changed, 22 insertions(+), 16 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 3f6d26c531..2ffcb073af 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -1672,6 +1672,26 @@ static void finalize_gic_version(VirtMachineState *vms)
}
}
+static void virt_cpu_post_init(VirtMachineState *vms)
+{
+bool aarch64;
+
+aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL);
+
+if (!kvm_enabled()) {
+if (aarch64 && vms->highmem) {
+int requested_pa_size = 64 - clz64(vms->highest_gpa);
+int pamax = arm_pamax(ARM_CPU(first_cpu));
+
+if (pamax < requested_pa_size) {
+error_report("VCPU supports less PA bits (%d) than requested "
+"by the memory map (%d)", pamax,
requested_pa_size);
+exit(1);
+}
+}
+ }
+}
+
static void machvirt_init(MachineState *machine)
{
VirtMachineState *vms = VIRT_MACHINE(machine);
@@ -1890,22 +1910,6 @@ static void machvirt_init(MachineState *machine)
fdt_add_timer_nodes(vms);
fdt_add_cpu_nodes(vms);
- if (!kvm_enabled()) {
-ARMCPU *cpu = ARM_CPU(first_cpu);
-bool aarch64 = object_property_get_bool(OBJECT(cpu), "aarch64", NULL);
-
-if (aarch64 && vms->highmem) {
-int requested_pa_size, pamax = arm_pamax(cpu);
-
-requested_pa_size = 64 - clz64(vms->highest_gpa);
-if (pamax < requested_pa_size) {
-error_report("VCPU supports less PA bits (%d) than requested "
-"by the memory map (%d)", pamax,
requested_pa_size);
-exit(1);
-}
-}
-}
-
memory_region_add_subregion(sysmem, vms->memmap[VIRT_MEM].base,
machine->ram);
if (machine->device_memory) {
@@ -1917,6 +1921,8 @@ static void machvirt_init(MachineState *machine)
create_gic(vms);
+virt_cpu_post_init(vms);
+
fdt_add_pmu_nodes(vms);
create_uart(vms, VIRT_UART, sysmem, serial_hd(0));
--
2.17.1
[PATCH v3 04/12] target/arm: spe: Only enable SPE from 5.2 compat machines.
Reviewed-by: Andrew Jones
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 7 +++
include/hw/arm/virt.h | 1 +
2 files changed, 8 insertions(+)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index acf9bfbece..3f6d26c531 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -1830,6 +1830,10 @@ static void machvirt_init(MachineState *machine)
object_property_set_bool(cpuobj, "pmu", false, NULL);
}
+if (vmc->no_spe && object_property_find(cpuobj, "spe", NULL)) {
+object_property_set_bool(cpuobj, "spe", false, NULL);
+}
+
if (object_property_find(cpuobj, "reset-cbar", NULL)) {
object_property_set_int(cpuobj, "reset-cbar",
vms->memmap[VIRT_CPUPERIPHS].base,
@@ -2553,8 +2557,11 @@ DEFINE_VIRT_MACHINE_AS_LATEST(5, 2)
static void virt_machine_5_1_options(MachineClass *mc)
{
+VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc));
+
virt_machine_5_2_options(mc);
compat_props_add(mc->compat_props, hw_compat_5_1, hw_compat_5_1_len);
+vmc->no_spe = true;
}
DEFINE_VIRT_MACHINE(5, 1)
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index dff67e1bef..72c269aaa5 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -126,6 +126,7 @@ typedef struct {
bool no_ged; /* Machines < 4.2 has no support for ACPI GED device */
bool kvm_no_adjvtime;
bool acpi_expose_flash;
+bool no_spe;
} VirtMachineClass;
typedef struct {
--
2.17.1
[PATCH v3 05/12] target/arm/kvm: spe: Unify device attr operation helper
From: Andrew Jones
Rename kvm_arm_pmu_set_attr() to kvm_arm_set_device_attr(),
So both the vPMU and vSPE device can share the same API.
Signed-off-by: Andrew Jones
Signed-off-by: Haibo Xu
---
target/arm/kvm64.c | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index ef1e960285..8ffd31ffdf 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -397,19 +397,20 @@ static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu,
target_ulong addr)
return NULL;
}
-static bool kvm_arm_pmu_set_attr(CPUState *cs, struct kvm_device_attr *attr)
+static bool kvm_arm_set_device_attr(CPUState *cs, struct kvm_device_attr *attr,
+const char *name)
{
int err;
err = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr);
if (err != 0) {
-error_report("PMU: KVM_HAS_DEVICE_ATTR: %s", strerror(-err));
+error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err));
return false;
}
err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, attr);
if (err != 0) {
-error_report("PMU: KVM_SET_DEVICE_ATTR: %s", strerror(-err));
+error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err));
return false;
}
@@ -426,7 +427,7 @@ void kvm_arm_pmu_init(CPUState *cs)
if (!ARM_CPU(cs)->has_pmu) {
return;
}
-if (!kvm_arm_pmu_set_attr(cs, )) {
+if (!kvm_arm_set_device_attr(cs, , "PMU")) {
error_report("failed to init PMU");
abort();
}
@@ -443,7 +444,7 @@ void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
if (!ARM_CPU(cs)->has_pmu) {
return;
}
-if (!kvm_arm_pmu_set_attr(cs, )) {
+if (!kvm_arm_set_device_attr(cs, , "PMU")) {
error_report("failed to set irq for PMU");
abort();
}
--
2.17.1
[PATCH v3 06/12] target/arm/kvm: spe: Add device init and set_irq operations
Reviewed-by: Andrew Jones
Signed-off-by: Haibo Xu
---
target/arm/kvm64.c | 33 +
target/arm/kvm_arm.h | 5 +
2 files changed, 38 insertions(+)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 8ffd31ffdf..5a2032fc9e 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -450,6 +450,39 @@ void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
}
}
+void kvm_arm_spe_init(CPUState *cs)
+{
+struct kvm_device_attr attr = {
+.group = KVM_ARM_VCPU_SPE_V1_CTRL,
+.attr = KVM_ARM_VCPU_SPE_V1_INIT,
+};
+
+if (!ARM_CPU(cs)->has_spe) {
+return;
+}
+if (!kvm_arm_set_device_attr(cs, , "SPE")) {
+error_report("failed to init SPE");
+abort();
+}
+}
+
+void kvm_arm_spe_set_irq(CPUState *cs, int irq)
+{
+struct kvm_device_attr attr = {
+.group = KVM_ARM_VCPU_SPE_V1_CTRL,
+.addr = (intptr_t),
+.attr = KVM_ARM_VCPU_SPE_V1_IRQ,
+};
+
+if (!ARM_CPU(cs)->has_spe) {
+return;
+}
+if (!kvm_arm_set_device_attr(cs, , "SPE")) {
+error_report("failed to set irq for SPE");
+abort();
+}
+}
+
static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id)
{
uint64_t ret;
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index f79655674e..bb155322eb 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -348,6 +348,8 @@ int kvm_arm_vgic_probe(void);
void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
void kvm_arm_pmu_init(CPUState *cs);
+void kvm_arm_spe_set_irq(CPUState *cs, int irq);
+void kvm_arm_spe_init(CPUState *cs);
int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level);
#else
@@ -397,6 +399,9 @@ static inline int kvm_arm_vgic_probe(void)
static inline void kvm_arm_pmu_set_irq(CPUState *cs, int irq) {}
static inline void kvm_arm_pmu_init(CPUState *cs) {}
+static inline void kvm_arm_spe_set_irq(CPUState *cs, int irq) {}
+static inline void kvm_arm_spe_init(CPUState *cs) {}
+
static inline void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) {}
static inline void kvm_arm_get_virtual_time(CPUState *cs) {}
--
2.17.1
[PATCH v3 12/12] target/arm: spe: Add corresponding test.
Reviewed-by: Andrew Jones
Signed-off-by: Haibo Xu
---
target/arm/monitor.c | 2 +-
tests/qtest/arm-cpu-features.c | 9 +
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/target/arm/monitor.c b/target/arm/monitor.c
index ba6e01abd0..1b8f08988a 100644
--- a/target/arm/monitor.c
+++ b/target/arm/monitor.c
@@ -99,7 +99,7 @@ QEMU_BUILD_BUG_ON(ARM_MAX_VQ > 16);
* then the order that considers those dependencies must be used.
*/
static const char *cpu_model_advertised_features[] = {
-"aarch64", "pmu", "sve",
+"aarch64", "pmu", "spe", "sve",
"sve128", "sve256", "sve384", "sve512",
"sve640", "sve768", "sve896", "sve1024", "sve1152", "sve1280",
"sve1408", "sve1536", "sve1664", "sve1792", "sve1920", "sve2048",
diff --git a/tests/qtest/arm-cpu-features.c b/tests/qtest/arm-cpu-features.c
index 77b5e30a9c..4d393fb2e2 100644
--- a/tests/qtest/arm-cpu-features.c
+++ b/tests/qtest/arm-cpu-features.c
@@ -494,6 +494,7 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
if (g_str_equal(qtest_get_arch(), "aarch64")) {
bool kvm_supports_sve;
+bool kvm_supports_spe;
char max_name[8], name[8];
uint32_t max_vq, vq;
uint64_t vls;
@@ -512,8 +513,10 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
"with KVM on this host", NULL);
assert_has_feature(qts, "host", "sve");
+assert_has_feature(qts, "host", "spe");
resp = do_query_no_props(qts, "host");
kvm_supports_sve = resp_get_feature(resp, "sve");
+kvm_supports_spe = resp_get_feature(resp, "spe");
vls = resp_get_sve_vls(resp);
qobject_unref(resp);
@@ -573,10 +576,16 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
} else {
g_assert(vls == 0);
}
+
+if (kvm_supports_spe) {
+assert_set_feature(qts, "host", "spe", false);
+assert_set_feature(qts, "host", "spe", true);
+}
} else {
assert_has_not_feature(qts, "host", "aarch64");
assert_has_not_feature(qts, "host", "pmu");
assert_has_not_feature(qts, "host", "sve");
+assert_has_not_feature(qts, "host", "spe");
}
qtest_quit(qts);
--
2.17.1
[PATCH v3 02/12] target/arm/kvm: spe: Add helper to detect SPE when using KVM
Reviewed-by: Andrew Jones
Reviewed-by: Richard Henderson
Signed-off-by: Haibo Xu
---
target/arm/kvm.c | 5 +
target/arm/kvm_arm.h | 13 +
2 files changed, 18 insertions(+)
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 8bb7318378..58f991e890 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -214,6 +214,11 @@ bool kvm_arm_pmu_supported(void)
return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3);
}
+bool kvm_arm_spe_supported(void)
+{
+return kvm_check_extension(kvm_state, KVM_CAP_ARM_SPE_V1);
+}
+
int kvm_arm_get_max_vm_ipa_size(MachineState *ms)
{
KVMState *s = KVM_STATE(ms->accelerator);
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index adb38514bf..f79655674e 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -283,6 +283,14 @@ bool kvm_arm_aarch32_supported(void);
*/
bool kvm_arm_pmu_supported(void);
+/**
+ * kvm_arm_spe_supported:
+ *
+ * Returns: true if the KVM VCPU can enable its SPE
+ * and false otherwise.
+ */
+bool kvm_arm_spe_supported(void);
+
/**
* kvm_arm_sve_supported:
*
@@ -366,6 +374,11 @@ static inline bool kvm_arm_pmu_supported(void)
return false;
}
+static inline bool kvm_arm_spe_supported(void)
+{
+return false;
+}
+
static inline bool kvm_arm_sve_supported(void)
{
return false;
--
2.17.1
[PATCH v3 11/12] target/arm/kvm: spe: Enable userspace irqchip support.
Since the current kernel patches haven't enabled the
userspace irqchip support, this patch is not verified yet!
Reviewed-by: Andrew Jones
Signed-off-by: Haibo Xu
---
hw/arm/virt.c| 2 +-
target/arm/kvm.c | 5 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index ee0831922b..bdb1ce925c 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -1715,8 +1715,8 @@ static void virt_cpu_post_init(VirtMachineState *vms)
assert(ARM_CPU(cpu)->has_spe == ON_OFF_AUTO_ON);
if (kvm_irqchip_in_kernel()) {
kvm_arm_spe_set_irq(cpu, PPI(VIRTUAL_SPE_IRQ));
-kvm_arm_spe_init(cpu);
}
+kvm_arm_spe_init(cpu);
}
}
} else {
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 58f991e890..7950ff1d83 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -820,6 +820,11 @@ MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run
*run)
switched_level &= ~KVM_ARM_DEV_PMU;
}
+if (switched_level & KVM_ARM_DEV_SPE) {
+qemu_set_irq(cpu->spe_interrupt,
+ !!(run->s.regs.device_irq_level & KVM_ARM_DEV_SPE));
+switched_level &= ~KVM_ARM_DEV_SPE;
+}
if (switched_level) {
qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
__func__, switched_level);
--
2.17.1
[PATCH v3 03/12] target/arm/cpu: spe: Add an option to turn on/off vSPE support
Adds a spe=[on/off] option to enable/disable vSPE support in
guest vCPU.
Reviewed-by: Andrew Jones
Signed-off-by: Haibo Xu
---
target/arm/cpu.c | 6 ++
target/arm/cpu.h | 13
target/arm/cpu64.c | 52 ++
3 files changed, 71 insertions(+)
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index c179e0752d..f211958eaa 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1310,6 +1310,12 @@ void arm_cpu_finalize_features(ARMCPU *cpu, Error **errp)
error_propagate(errp, local_err);
return;
}
+
+arm_cpu_spe_finalize(cpu, _err);
+if (local_err != NULL) {
+error_propagate(errp, local_err);
+return;
+}
}
}
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index a1c7d8ebae..baf2bbcee8 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -24,6 +24,7 @@
#include "hw/registerfields.h"
#include "cpu-qom.h"
#include "exec/cpu-defs.h"
+#include "qapi/qapi-types-common.h"
/* ARM processors have a weak memory model */
#define TCG_GUEST_DEFAULT_MO (0)
@@ -196,9 +197,11 @@ typedef struct {
#ifdef TARGET_AARCH64
# define ARM_MAX_VQ16
void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp);
+void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp);
#else
# define ARM_MAX_VQ1
static inline void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) { }
+static inline void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp) { }
#endif
typedef struct ARMVectorReg {
@@ -829,6 +832,8 @@ struct ARMCPU {
bool has_el3;
/* CPU has PMU (Performance Monitor Unit) */
bool has_pmu;
+/* CPU has SPE (Statistical Profiling Extension) */
+OnOffAuto has_spe;
/* CPU has VFP */
bool has_vfp;
/* CPU has Neon */
@@ -3869,6 +3874,14 @@ static inline bool isar_feature_aa64_pmu_8_4(const
ARMISARegisters *id)
FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) != 0xf;
}
+/*
+ * Currently we don't differentiate between the ARMv8.2-SPE and ARMv8.3-SPE.
+ */
+static inline bool isar_feature_aa64_spe(const ARMISARegisters *id)
+{
+return FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMSVER) != 0;
+}
+
static inline bool isar_feature_aa64_rcpc_8_3(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, LRCPC) != 0;
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 3c2b3d9599..4997c4a3c0 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -572,6 +572,55 @@ void aarch64_add_sve_properties(Object *obj)
}
}
+void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp)
+{
+uint64_t t;
+uint32_t value = 0;
+
+if (cpu->has_spe == ON_OFF_AUTO_AUTO) {
+if (kvm_enabled() && kvm_arm_spe_supported()) {
+cpu->has_spe = ON_OFF_AUTO_ON;
+} else {
+cpu->has_spe = ON_OFF_AUTO_OFF;
+}
+} else if (cpu->has_spe == ON_OFF_AUTO_ON) {
+if (!kvm_enabled() || !kvm_arm_spe_supported()) {
+error_setg(errp, "'spe' cannot be enabled on this host");
+return;
+}
+}
+
+/*
+ * According to the ARM ARM, the ID_AA64DFR0[PMSVER] currently
+ * support 3 values:
+ *
+ * 0b: SPE not implemented
+ * 0b0001: ARMv8.2-SPE implemented
+ * 0b0010: ARMv8.3-SPE implemented
+ *
+ * But the kernel KVM API didn't expose all these 3 values, and
+ * we can only get whether the SPE feature is supported or not.
+ * So here we just set the PMSVER to 1 if this feature was supported.
+ */
+if (cpu->has_spe == ON_OFF_AUTO_ON) {
+value = 1;
+}
+
+t = cpu->isar.id_aa64dfr0;
+t = FIELD_DP64(t, ID_AA64DFR0, PMSVER, value);
+cpu->isar.id_aa64dfr0 = t;
+}
+
+static bool arm_spe_get(Object *obj, Error **errp)
+{
+return ARM_CPU(obj)->has_spe != ON_OFF_AUTO_OFF;
+}
+
+static void arm_spe_set(Object *obj, bool value, Error **errp)
+{
+ARM_CPU(obj)->has_spe = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
+}
+
/* -cpu max: if KVM is enabled, like -cpu host (best possible with this host);
* otherwise, a CPU with as many features enabled as our emulation supports.
* The version of '-cpu max' for qemu-system-arm is defined in cpu.c;
@@ -721,6 +770,9 @@ static void aarch64_max_initfn(Object *obj)
aarch64_add_sve_properties(obj);
object_property_add(obj, "sve-max-vq", "uint32", cpu_max_get_sve_max_vq,
cpu_max_set_sve_max_vq, NULL, NULL);
+
+cpu->has_spe = ON_OFF_AUTO_AUTO;
+object_property_add_bool(obj, "spe", arm_spe_get, arm_spe_set);
}
static const ARMCPUInfo aarch64_cpus[] = {
--
2.17.1
[PATCH v3 01/12] update Linux headers with new vSPE macros
Signed-off-by: Haibo Xu
---
linux-headers/asm-arm64/kvm.h | 4
linux-headers/linux/kvm.h | 2 ++
2 files changed, 6 insertions(+)
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index 9e34f0f875..802319ee02 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -106,6 +106,7 @@ struct kvm_regs {
#define KVM_ARM_VCPU_SVE 4 /* enable SVE for this CPU */
#define KVM_ARM_VCPU_PTRAUTH_ADDRESS 5 /* VCPU uses address authentication */
#define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */
+#define KVM_ARM_VCPU_SPE_V17 /* Support guest SPEv1 */
struct kvm_vcpu_init {
__u32 target;
@@ -334,6 +335,9 @@ struct kvm_vcpu_events {
#define KVM_ARM_VCPU_TIMER_IRQ_PTIMER1
#define KVM_ARM_VCPU_PVTIME_CTRL 2
#define KVM_ARM_VCPU_PVTIME_IPA 0
+#define KVM_ARM_VCPU_SPE_V1_CTRL 3
+#define KVM_ARM_VCPU_SPE_V1_IRQ 0
+#define KVM_ARM_VCPU_SPE_V1_INIT 1
/* KVM_IRQ_LINE irq field index values */
#define KVM_ARM_IRQ_VCPU2_SHIFT28
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index a28c366737..35ef0ae842 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1031,6 +1031,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_PPC_SECURE_GUEST 181
#define KVM_CAP_HALT_POLL 182
#define KVM_CAP_ASYNC_PF_INT 183
+#define KVM_CAP_ARM_SPE_V1 184
#ifdef KVM_CAP_IRQ_ROUTING
@@ -1671,6 +1672,7 @@ struct kvm_assigned_msix_entry {
#define KVM_ARM_DEV_EL1_VTIMER (1 << 0)
#define KVM_ARM_DEV_EL1_PTIMER (1 << 1)
#define KVM_ARM_DEV_PMU(1 << 2)
+#define KVM_ARM_DEV_SPE(1 << 3)
struct kvm_hyperv_eventfd {
__u32 conn_id;
--
2.17.1
[PATCH v3 00/12] target/arm: Add vSPE support to KVM guest
v3: - Moved the patch 11 kernel header change to patch 01 - Removed doc for vSPE - Moved the assignment of has_spe into aarch64_add_spe_properties - Only enable spe properties in KVM mode - Only trigger the kvm_arm_spe_init with in-kernel irqchip support to avoid breaking bisection - Set the PMSVER bit in ID register if spe was enabled by usespace Again, many thanks to Andrew, Richard, Philippe and Auger for their comments. This series add support for SPE(Statistical Profiling Extension)[1] in KVM guest. It's based on Andrew Murray's kernel KVM patches V2[2], and has been tested to ensure that guest can use SPE with valid data. E.g. In host: $ ./qemu-system-aarch64 \ -cpu host -M virt,accel=kvm,gic-version=3 -nographic -m 2048M \ -kernel ./Image-new \ -initrd /boot/initrd.img-5.6.0-rc2+ \ -append "root=/dev/vda rw console=ttyAMA0" -nodefaults -serial stdio\ -drive if=none,file=./xenial.rootfs.ext4,id=hd0,format=raw \ -device virtio-blk-device,drive=hd0 \ In guest: $ perf record -e arm_spe/ts_enable=1,pa_enable=1,pct_enable=1/ \ dd if=/dev/zero of=/dev/null count=1000 $ perf report --dump-raw-trace > spe_buf.txt The spe_buf.txt should contain similar data as below: . ... ARM SPE data: size 135944 bytes . : b0 f4 d3 29 10 00 80 ff a0 PC 0xff80001029d3f4 el1 ns=1 . 0009: 99 0b 00LAT 11 ISSUE . 000c: 98 0d 00LAT 13 TOT . 000f: 52 16 00EV RETIRED L1D-ACCESS TLB-ACCESS . 0012: 49 00 LD . 0014: b2 d0 40 d8 70 00 00 ff 00 VA 0xff70d840d0 . 001d: 9a 01 00LAT 1 XLAT . 0020: 00 00 00PAD . 0023: 71 a5 1f b3 20 14 00 00 00 TS 86447955877 . 002c: b0 7c f9 29 10 00 80 ff a0 PC 0xff80001029f97c el1 ns=1 . 0035: 99 02 00LAT 2 ISSUE . 0038: 98 03 00LAT 3 TOT . 003b: 52 02 00EV RETIRED . 003e: 48 00 INSN-OTHER . 0040: 00 00 00PAD . 0043: 71 ef 1f b3 20 14 00 00 00 TS 86447955951 . 004c: b0 f0 e9 29 10 00 80 ff a0 PC 0xff80001029e9f0 el1 ns=1 . 0055: 99 02 00LAT 2 ISSUE . 0058: 98 03 00LAT 3 TOT . 005b: 52 02 00EV RETIRED If you want to disable the vSPE support, you can use the 'spe=off' cpu property: ./qemu-system-aarch64 \ -cpu host,spe=off -M virt,accel=kvm,gic-version=3 -nographic -m 2048M \ -kernel ./Image-new \ -initrd /boot/initrd.img-5.6.0-rc2+ \ -append "root=/dev/vda rw console=ttyAMA0" -nodefaults -serial stdio\ -drive if=none,file=./xenial.rootfs.ext4,id=hd0,format=raw \ -device virtio-blk-device,drive=hd0 \ Note: (1) Since the kernel patches are still under review, some of the macros in the header files may be changed after merging. We may need to update them accordingly. To be specific, if you want to have a try on this patch series, you needs to replace on the kernel 5.5-rc2 based series, and do minor changes: -#define KVM_CAP_ARM_SPE_V1 179 +#define KVM_CAP_ARM_SPE_V1 184 (2) These patches only add vSPE support in KVM mode, for TCG mode, I'm not sure whether we need to support it. (3) Just followed the 'sve' property, we only allow this feature to be removed from CPUs which enable it by default when the host cpu support it. [1]https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/ posts/statistical-profiling-extension-for-armv8-a [2]https://www.spinics.net/lists/arm-kernel/msg776228.html [3]https://www.mail-archive.com/qemu-devel@nongnu.org/msg727588.html Haibo Xu (12): update Linux headers with new vSPE macros target/arm/kvm: spe: Add helper to detect SPE when using KVM target/arm/cpu: spe: Add an option to turn on/off vSPE support target/arm: spe: Only enable SPE from 5.2 compat machines. target/arm/kvm: spe: Unify device attr operation helper target/arm/kvm: spe: Add device init and set_irq operations hw/arm/virt: Move post cpu realize check into its own function hw/arm/virt: Move kvm pmu setup to virt_cpu_post_init hw/arm/virt: spe: Add vSPE device and corresponding interrupt support target/arm/cpu: spe: Enable spe to work with host cpu target/arm/kvm: spe: Enable userspace irqchip support. target/arm: spe: Add corresponding test. hw/arm/virt-acpi-bu
Re: [PATCH v2 09/12] hw/arm/virt: spe: Add SPE fdt binding for virt machine
On Wed, 9 Sep 2020 at 17:34, Andrew Jones wrote:
>
> On Wed, Sep 09, 2020 at 03:51:14PM +0800, Haibo Xu wrote:
> > > > +
> > > > +if (spe) {
> > > > +assert(ARM_CPU(cpu)->has_spe == ON_OFF_AUTO_ON);
> > > > +if (kvm_irqchip_in_kernel()) {
> > > > +kvm_arm_spe_set_irq(cpu, PPI(VIRTUAL_SPE_IRQ));
> > > > +}
> > > > +kvm_arm_spe_init(cpu);
> > >
> > > A later patch introduces userspace irqchip support. Should we avoid
> > > allowing it until then to avoid breaking bisection?
> > >
> >
> > Yes, it's possible to break the bisection. To avoid it I think we can
> > move the above codes
> > block to a separate patch after adding the userspace irqchip support,
> > Or, just put the userspace
> > irqchip support patch before this patch. What's your opinion?
> >
>
> This patch ca forbid SPE without kernel irqchip. Then the patch that adds
> userspace irqchip support would also remove the restriction.
>
> Thanks,
> drew
>
Good idea! Will fix it in v3.
Re: [PATCH v2 10/12] target/arm/cpu: spe: Enable spe to work with host cpu
On Tue, 8 Sep 2020 at 19:33, Andrew Jones wrote:
>
> On Tue, Sep 08, 2020 at 08:13:28AM +, Haibo Xu wrote:
> > Turn on the spe cpu property by default if host cpu
> > support it, i.e. we can now do '-cpu max|host' to add
> > the vSPE, and '-cpu max|host,spe=off' to remove it.
> >
> > Signed-off-by: Haibo Xu
> > ---
> > target/arm/cpu.c | 3 +++
> > target/arm/cpu.h | 2 ++
> > target/arm/cpu64.c | 7 ++-
> > target/arm/kvm64.c | 12
> > 4 files changed, 23 insertions(+), 1 deletion(-)
> >
> > diff --git a/target/arm/cpu.c b/target/arm/cpu.c
> > index 786cc6134c..58f12d6eb5 100644
> > --- a/target/arm/cpu.c
> > +++ b/target/arm/cpu.c
> > @@ -2271,6 +2271,9 @@ static void arm_host_initfn(Object *obj)
> > kvm_arm_set_cpu_features_from_host(cpu);
> > if (arm_feature(>env, ARM_FEATURE_AARCH64)) {
> > aarch64_add_sve_properties(obj);
> > +
> > +cpu->has_spe = ON_OFF_AUTO_AUTO;
> > +aarch64_add_spe_properties(obj);
>
> Why not put the assignment of has_spe into aarch64_add_spe_properties()?
>
Yes, it could be. Will fix it in v3.
> > }
> > arm_cpu_post_init(obj);
> > }
> > diff --git a/target/arm/cpu.h b/target/arm/cpu.h
> > index 395a1e5df8..5a3ea876c8 100644
> > --- a/target/arm/cpu.h
> > +++ b/target/arm/cpu.h
> > @@ -1040,6 +1040,7 @@ void aarch64_sve_narrow_vq(CPUARMState *env, unsigned
> > vq);
> > void aarch64_sve_change_el(CPUARMState *env, int old_el,
> > int new_el, bool el0_a64);
> > void aarch64_add_sve_properties(Object *obj);
> > +void aarch64_add_spe_properties(Object *obj);
> >
> > /*
> > * SVE registers are encoded in KVM's memory in an endianness-invariant
> > format.
> > @@ -1071,6 +1072,7 @@ static inline void aarch64_sve_change_el(CPUARMState
> > *env, int o,
> > int n, bool a)
> > { }
> > static inline void aarch64_add_sve_properties(Object *obj) { }
> > +static inline void aarch64_add_spe_properties(Object *obj) { }
> > #endif
> >
> > #if !defined(CONFIG_TCG)
> > diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
> > index 4997c4a3c0..d38c55e2ca 100644
> > --- a/target/arm/cpu64.c
> > +++ b/target/arm/cpu64.c
> > @@ -621,6 +621,11 @@ static void arm_spe_set(Object *obj, bool value, Error
> > **errp)
> > ARM_CPU(obj)->has_spe = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
> > }
> >
> > +void aarch64_add_spe_properties(Object *obj)
> > +{
> > +object_property_add_bool(obj, "spe", arm_spe_get, arm_spe_set);
> > +}
> > +
> > /* -cpu max: if KVM is enabled, like -cpu host (best possible with this
> > host);
> > * otherwise, a CPU with as many features enabled as our emulation
> > supports.
> > * The version of '-cpu max' for qemu-system-arm is defined in cpu.c;
> > @@ -772,7 +777,7 @@ static void aarch64_max_initfn(Object *obj)
> > cpu_max_set_sve_max_vq, NULL, NULL);
> >
> > cpu->has_spe = ON_OFF_AUTO_AUTO;
> > -object_property_add_bool(obj, "spe", arm_spe_get, arm_spe_set);
> > +aarch64_add_spe_properties(obj);
>
> If TCG doesn't support this cpu feature then this should be in the
> kvm_enabled() part of this function.
>
>
Will fix it in v3.
> > }
> >
> > static const ARMCPUInfo aarch64_cpus[] = {
> > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> > index 5a2032fc9e..3f0a09c05b 100644
> > --- a/target/arm/kvm64.c
> > +++ b/target/arm/kvm64.c
> > @@ -515,6 +515,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> > *ahcf)
> > */
> > int fdarray[3];
> > bool sve_supported;
> > +bool spe_supported;
> > uint64_t features = 0;
> > uint64_t t;
> > int err;
> > @@ -655,6 +656,8 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> > *ahcf)
> > }
> >
> > sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION,
> > KVM_CAP_ARM_SVE) > 0;
> > +spe_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION,
> > + KVM_CAP_ARM_SPE_V1) > 0;
> >
> > kvm_arm_destroy_scratch_host_vcpu(fdarray);
> >
> > @@ -668,6 +671,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
> > *ahcf)
> > t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
> > ahcf->isar.id_aa64pfr0 = t;
> > }
>
Re: [PATCH v2 09/12] hw/arm/virt: spe: Add SPE fdt binding for virt machine
On Tue, 8 Sep 2020 at 19:16, Andrew Jones wrote:
>
>
> This patch does much more than the summary "hw/arm/virt: spe: Add SPE fdt
> binding for virt machine" says it does. Please revise the summary.
>
Will revise it in v3.
Thanks,
Haibo
> On Tue, Sep 08, 2020 at 08:13:27AM +, Haibo Xu wrote:
> > Add a virtual SPE device for virt machine while using
> > PPI 5 for SPE overflow interrupt number which has already
> > selected in kvmtool.
> >
> > Signed-off-by: Haibo Xu
> > ---
> > hw/arm/virt-acpi-build.c| 3 +++
> > hw/arm/virt.c | 43 -
> > include/hw/acpi/acpi-defs.h | 3 +++
> > include/hw/arm/virt.h | 1 +
> > target/arm/cpu.c| 2 ++
> > target/arm/cpu.h| 2 ++
> > 6 files changed, 53 insertions(+), 1 deletion(-)
> >
> > diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
> > index 9efd7a3881..3fd80fda53 100644
> > --- a/hw/arm/virt-acpi-build.c
> > +++ b/hw/arm/virt-acpi-build.c
> > @@ -665,6 +665,9 @@ build_madt(GArray *table_data, BIOSLinker *linker,
> > VirtMachineState *vms)
> > if (arm_feature(>env, ARM_FEATURE_PMU)) {
> > gicc->performance_interrupt =
> > cpu_to_le32(PPI(VIRTUAL_PMU_IRQ));
> > }
> > +if (cpu_isar_feature(aa64_spe, armcpu)) {
> > +gicc->spe_interrupt = cpu_to_le32(PPI(VIRTUAL_SPE_IRQ));
> > +}
> > if (vms->virt) {
> > gicc->vgic_interrupt = cpu_to_le32(PPI(ARCH_GIC_MAINT_IRQ));
> > }
> > diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> > index 6bacfb668d..bdb1ce925c 100644
> > --- a/hw/arm/virt.c
> > +++ b/hw/arm/virt.c
> > @@ -545,6 +545,32 @@ static void fdt_add_pmu_nodes(const VirtMachineState
> > *vms)
> > }
> > }
> >
> > +static void fdt_add_spe_nodes(const VirtMachineState *vms)
> > +{
> > +ARMCPU *armcpu = ARM_CPU(first_cpu);
> > +uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
> > +
> > +if (!cpu_isar_feature(aa64_spe, armcpu)) {
> > +assert(!object_property_get_bool(OBJECT(armcpu), "spe", NULL));
> > +return;
> > +}
> > +
> > +if (vms->gic_version == VIRT_GIC_VERSION_2) {
> > +irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
> > + GIC_FDT_IRQ_PPI_CPU_WIDTH,
> > + (1 << vms->smp_cpus) - 1);
> > +}
> > +
> > +qemu_fdt_add_subnode(vms->fdt, "/spe");
> > +if (arm_feature(>env, ARM_FEATURE_V8)) {
> > +const char compat[] = "arm,statistical-profiling-extension-v1";
> > +qemu_fdt_setprop(vms->fdt, "/spe", "compatible",
> > + compat, sizeof(compat));
> > +qemu_fdt_setprop_cells(vms->fdt, "/spe", "interrupts",
> > + GIC_FDT_IRQ_TYPE_PPI, VIRTUAL_SPE_IRQ,
> > irqflags);
> > +}
> > +}
> > +
> > static inline DeviceState *create_acpi_ged(VirtMachineState *vms)
> > {
> > DeviceState *dev;
> > @@ -717,6 +743,10 @@ static void create_gic(VirtMachineState *vms)
> > qdev_get_gpio_in(vms->gic, ppibase
> > + VIRTUAL_PMU_IRQ));
> >
> > +qdev_connect_gpio_out_named(cpudev, "spe-interrupt", 0,
> > +qdev_get_gpio_in(vms->gic, ppibase
> > + + VIRTUAL_SPE_IRQ));
> > +
> > sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev,
> > ARM_CPU_IRQ));
> > sysbus_connect_irq(gicbusdev, i + smp_cpus,
> > qdev_get_gpio_in(cpudev, ARM_CPU_FIQ));
> > @@ -1664,11 +1694,12 @@ static void finalize_gic_version(VirtMachineState
> > *vms)
> >
> > static void virt_cpu_post_init(VirtMachineState *vms)
> > {
> > -bool aarch64, pmu;
> > +bool aarch64, pmu, spe;
> > CPUState *cpu;
> >
> > aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL);
> > pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL);
> > +spe = object_property_get_bool(OBJECT(first_cpu), "spe", NULL);
> >
> > if (kvm_enabled()) {
> > CPU_FOREACH(cpu) {
> > @@ -1679,6 +1710,14 @@ static vo
Re: [PATCH v2 07/12] hw/arm/virt: Move post cpu realize check into its own function
On Tue, 8 Sep 2020 at 19:03, Andrew Jones wrote: > > On Tue, Sep 08, 2020 at 08:13:25AM +, Haibo Xu wrote: > > From: Andrew Jones > > > > We'll add more to this new function in coming patches so we also > > state the gic must be created and call it below create_gic(). > > > > No functional change intended. > > > > Signed-off-by: Andrew Jones > > This is the wrong version of this patch. You should use v2, for which > Peter has given his r-b > > Thanks, > drew > Will fix it in v3. Thanks, Haibo
Re: [PATCH v2 12/12] target/arm: spe: Add corresponding doc and test.
On Tue, 8 Sep 2020 at 19:41, Andrew Jones wrote:
>
> On Tue, Sep 08, 2020 at 08:13:30AM +, Haibo Xu wrote:
> > Signed-off-by: Haibo Xu
> > ---
> > docs/system/arm/cpu-features.rst | 20
> > target/arm/monitor.c | 2 +-
> > tests/qtest/arm-cpu-features.c | 9 +
> > 3 files changed, 30 insertions(+), 1 deletion(-)
> >
> > diff --git a/docs/system/arm/cpu-features.rst
> > b/docs/system/arm/cpu-features.rst
> > index 2d5c06cd01..5b81b9a560 100644
> > --- a/docs/system/arm/cpu-features.rst
> > +++ b/docs/system/arm/cpu-features.rst
> > @@ -344,3 +344,23 @@ verbose command lines. However, the recommended way
> > to select vector
> > lengths is to explicitly enable each desired length. Therefore only
> > example's (1), (4), and (6) exhibit recommended uses of the properties.
> >
> > +SPE CPU Property
> > +==
>
> Too many '='
>
> > +
> > +The SPE CPU property `spe` is used to enable or disable the SPE feature,
> > +just as the `pmu` CPU property completely enables or disables the PMU.
> > +
> > +Currently, this property is only available with KVM mode, and is enabled
> > +by default if KVM support it. When KVM is enabled, if the host does not
> > +support SPE, then an error is generated when attempting to enable it.
> > +
> > +Following are 2 examples to use this property:
> > +
> > + 1) Disable SPE::
> > +
> > + $ qemu-system-aarch64 -M virt,accel=kvm -cpu max,spe=off
> > +
> > + 2) Implicitly enable it with the `host` CPU type if host cpu
> > + support it::
>
> if the host CPU supports it
>
>
> Actually, I'm not sure we need to document this feature. We didn't bother
> documenting pauth, since there wasn't anything special about it and
> there's nothing special about this feature either.
>
Yes, there is no special treatment for this feature, and it just
follows the syntax
of other vCPU features. Will remove this doc in v3.
Anyway, thanks so much for the review!
Regards,
Haibo
> > +
> > + $ qemu-system-aarch64 -M virt,accel=kvm -cpu host
> > diff --git a/target/arm/monitor.c b/target/arm/monitor.c
> > index ba6e01abd0..1b8f08988a 100644
> > --- a/target/arm/monitor.c
> > +++ b/target/arm/monitor.c
> > @@ -99,7 +99,7 @@ QEMU_BUILD_BUG_ON(ARM_MAX_VQ > 16);
> > * then the order that considers those dependencies must be used.
> > */
> > static const char *cpu_model_advertised_features[] = {
> > -"aarch64", "pmu", "sve",
> > +"aarch64", "pmu", "spe", "sve",
> > "sve128", "sve256", "sve384", "sve512",
> > "sve640", "sve768", "sve896", "sve1024", "sve1152", "sve1280",
> > "sve1408", "sve1536", "sve1664", "sve1792", "sve1920", "sve2048",
> > diff --git a/tests/qtest/arm-cpu-features.c b/tests/qtest/arm-cpu-features.c
> > index 77b5e30a9c..4d393fb2e2 100644
> > --- a/tests/qtest/arm-cpu-features.c
> > +++ b/tests/qtest/arm-cpu-features.c
> > @@ -494,6 +494,7 @@ static void test_query_cpu_model_expansion_kvm(const
> > void *data)
> >
> > if (g_str_equal(qtest_get_arch(), "aarch64")) {
> > bool kvm_supports_sve;
> > +bool kvm_supports_spe;
> > char max_name[8], name[8];
> > uint32_t max_vq, vq;
> > uint64_t vls;
> > @@ -512,8 +513,10 @@ static void test_query_cpu_model_expansion_kvm(const
> > void *data)
> > "with KVM on this host", NULL);
> >
> > assert_has_feature(qts, "host", "sve");
> > +assert_has_feature(qts, "host", "spe");
> > resp = do_query_no_props(qts, "host");
> > kvm_supports_sve = resp_get_feature(resp, "sve");
> > +kvm_supports_spe = resp_get_feature(resp, "spe");
> > vls = resp_get_sve_vls(resp);
> > qobject_unref(resp);
> >
> > @@ -573,10 +576,16 @@ static void test_query_cpu_model_expansion_kvm(const
> > void *data)
> > } else {
> > g_assert(vls == 0);
> > }
> > +
> > +if (kvm_supports_spe) {
> > +assert_set_feature(qts, "host", "spe", false);
> > +assert_set_feature(qts, "host", "spe", true);
> > +}
> > } else {
> > assert_has_not_feature(qts, "host", "aarch64");
> > assert_has_not_feature(qts, "host", "pmu");
> > assert_has_not_feature(qts, "host", "sve");
> > +assert_has_not_feature(qts, "host", "spe");
> > }
> >
> > qtest_quit(qts);
> > --
> > 2.17.1
> >
>
> Otherwise
>
> Reviewed-by: Andrew Jones
>
Re: [PATCH v2 11/12] target/arm/kvm: spe: Enable userspace irqchip support.
On Tue, 8 Sep 2020 at 19:35, Andrew Jones wrote:
>
> On Tue, Sep 08, 2020 at 08:13:29AM +, Haibo Xu wrote:
> > Since the current kernel patches haven't enabled the
> > userspace irqchip support, this patch is not verified yet!
> >
> > Signed-off-by: Haibo Xu
> > ---
> > linux-headers/linux/kvm.h | 1 +
> > target/arm/kvm.c | 5 +
> > 2 files changed, 6 insertions(+)
> >
> > diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
> > index 8840cbb01c..35ef0ae842 100644
> > --- a/linux-headers/linux/kvm.h
> > +++ b/linux-headers/linux/kvm.h
> > @@ -1672,6 +1672,7 @@ struct kvm_assigned_msix_entry {
> > #define KVM_ARM_DEV_EL1_VTIMER (1 << 0)
> > #define KVM_ARM_DEV_EL1_PTIMER (1 << 1)
> > #define KVM_ARM_DEV_PMU (1 << 2)
> > +#define KVM_ARM_DEV_SPE (1 << 3)
>
> kernel header changes should be separate patches
>
Will move this line to patch 01 in v3.
Thanks,
Haibo
> >
> > struct kvm_hyperv_eventfd {
> > __u32 conn_id;
> > diff --git a/target/arm/kvm.c b/target/arm/kvm.c
> > index 58f991e890..7950ff1d83 100644
> > --- a/target/arm/kvm.c
> > +++ b/target/arm/kvm.c
> > @@ -820,6 +820,11 @@ MemTxAttrs kvm_arch_post_run(CPUState *cs, struct
> > kvm_run *run)
> > switched_level &= ~KVM_ARM_DEV_PMU;
> > }
> >
> > +if (switched_level & KVM_ARM_DEV_SPE) {
> > +qemu_set_irq(cpu->spe_interrupt,
> > + !!(run->s.regs.device_irq_level &
> > KVM_ARM_DEV_SPE));
> > +switched_level &= ~KVM_ARM_DEV_SPE;
> > +}
> > if (switched_level) {
> > qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ
> > %x\n",
> >__func__, switched_level);
> > --
> > 2.17.1
> >
>
> Otherwise
>
> Reviewed-by: Andrew Jones
>
Re: [PATCH v2 05/12] target/arm/kvm: spe: Unify device attr operation helper
On Tue, 8 Sep 2020 at 18:56, Andrew Jones wrote:
>
> On Tue, Sep 08, 2020 at 08:13:23AM +, Haibo Xu wrote:
> > From: Andrew Jones
> >
> > Rename kvm_arm_pmu_set_attr() to kvm_arm_set_device_attr(),
> > So both the vPMU and vSPE device can share the same API.
> >
> > Signed-off-by: Andrew Jones
>
> Looks like a faithful port of what I posted as a hunk of another patch, so
> I'll accept the authorship. Please also add you s-b though.
>
> Thanks,
> drew
>
Ok, will fix it in v3.
Thanks,
Haibo
> > ---
> > target/arm/kvm64.c | 11 ++-
> > 1 file changed, 6 insertions(+), 5 deletions(-)
> >
> > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> > index ef1e960285..8ffd31ffdf 100644
> > --- a/target/arm/kvm64.c
> > +++ b/target/arm/kvm64.c
> > @@ -397,19 +397,20 @@ static CPUWatchpoint *find_hw_watchpoint(CPUState
> > *cpu, target_ulong addr)
> > return NULL;
> > }
> >
> > -static bool kvm_arm_pmu_set_attr(CPUState *cs, struct kvm_device_attr
> > *attr)
> > +static bool kvm_arm_set_device_attr(CPUState *cs, struct kvm_device_attr
> > *attr,
> > +const char *name)
> > {
> > int err;
> >
> > err = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr);
> > if (err != 0) {
> > -error_report("PMU: KVM_HAS_DEVICE_ATTR: %s", strerror(-err));
> > +error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err));
> > return false;
> > }
> >
> > err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, attr);
> > if (err != 0) {
> > -error_report("PMU: KVM_SET_DEVICE_ATTR: %s", strerror(-err));
> > +error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err));
> > return false;
> > }
> >
> > @@ -426,7 +427,7 @@ void kvm_arm_pmu_init(CPUState *cs)
> > if (!ARM_CPU(cs)->has_pmu) {
> > return;
> > }
> > -if (!kvm_arm_pmu_set_attr(cs, )) {
> > +if (!kvm_arm_set_device_attr(cs, , "PMU")) {
> > error_report("failed to init PMU");
> > abort();
> > }
> > @@ -443,7 +444,7 @@ void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
> > if (!ARM_CPU(cs)->has_pmu) {
> > return;
> > }
> > -if (!kvm_arm_pmu_set_attr(cs, )) {
> > +if (!kvm_arm_set_device_attr(cs, , "PMU")) {
> > error_report("failed to set irq for PMU");
> > abort();
> > }
> > --
> > 2.17.1
> >
> >
>
[PATCH v2 06/12] target/arm/kvm: spe: Add device init and set_irq operations
Signed-off-by: Haibo Xu
---
target/arm/kvm64.c | 33 +
target/arm/kvm_arm.h | 5 +
2 files changed, 38 insertions(+)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 8ffd31ffdf..5a2032fc9e 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -450,6 +450,39 @@ void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
}
}
+void kvm_arm_spe_init(CPUState *cs)
+{
+struct kvm_device_attr attr = {
+.group = KVM_ARM_VCPU_SPE_V1_CTRL,
+.attr = KVM_ARM_VCPU_SPE_V1_INIT,
+};
+
+if (!ARM_CPU(cs)->has_spe) {
+return;
+}
+if (!kvm_arm_set_device_attr(cs, , "SPE")) {
+error_report("failed to init SPE");
+abort();
+}
+}
+
+void kvm_arm_spe_set_irq(CPUState *cs, int irq)
+{
+struct kvm_device_attr attr = {
+.group = KVM_ARM_VCPU_SPE_V1_CTRL,
+.addr = (intptr_t),
+.attr = KVM_ARM_VCPU_SPE_V1_IRQ,
+};
+
+if (!ARM_CPU(cs)->has_spe) {
+return;
+}
+if (!kvm_arm_set_device_attr(cs, , "SPE")) {
+error_report("failed to set irq for SPE");
+abort();
+}
+}
+
static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id)
{
uint64_t ret;
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index f79655674e..bb155322eb 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -348,6 +348,8 @@ int kvm_arm_vgic_probe(void);
void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
void kvm_arm_pmu_init(CPUState *cs);
+void kvm_arm_spe_set_irq(CPUState *cs, int irq);
+void kvm_arm_spe_init(CPUState *cs);
int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level);
#else
@@ -397,6 +399,9 @@ static inline int kvm_arm_vgic_probe(void)
static inline void kvm_arm_pmu_set_irq(CPUState *cs, int irq) {}
static inline void kvm_arm_pmu_init(CPUState *cs) {}
+static inline void kvm_arm_spe_set_irq(CPUState *cs, int irq) {}
+static inline void kvm_arm_spe_init(CPUState *cs) {}
+
static inline void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) {}
static inline void kvm_arm_get_virtual_time(CPUState *cs) {}
--
2.17.1
[PATCH v2 11/12] target/arm/kvm: spe: Enable userspace irqchip support.
Since the current kernel patches haven't enabled the
userspace irqchip support, this patch is not verified yet!
Signed-off-by: Haibo Xu
---
linux-headers/linux/kvm.h | 1 +
target/arm/kvm.c | 5 +
2 files changed, 6 insertions(+)
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 8840cbb01c..35ef0ae842 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1672,6 +1672,7 @@ struct kvm_assigned_msix_entry {
#define KVM_ARM_DEV_EL1_VTIMER (1 << 0)
#define KVM_ARM_DEV_EL1_PTIMER (1 << 1)
#define KVM_ARM_DEV_PMU(1 << 2)
+#define KVM_ARM_DEV_SPE(1 << 3)
struct kvm_hyperv_eventfd {
__u32 conn_id;
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 58f991e890..7950ff1d83 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -820,6 +820,11 @@ MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run
*run)
switched_level &= ~KVM_ARM_DEV_PMU;
}
+if (switched_level & KVM_ARM_DEV_SPE) {
+qemu_set_irq(cpu->spe_interrupt,
+ !!(run->s.regs.device_irq_level & KVM_ARM_DEV_SPE));
+switched_level &= ~KVM_ARM_DEV_SPE;
+}
if (switched_level) {
qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
__func__, switched_level);
--
2.17.1
[PATCH v2 04/12] target/arm: spe: Only enable SPE from 5.2 compat machines.
Signed-off-by: Haibo Xu
---
hw/arm/virt.c | 7 +++
include/hw/arm/virt.h | 1 +
2 files changed, 8 insertions(+)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index acf9bfbece..3f6d26c531 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -1830,6 +1830,10 @@ static void machvirt_init(MachineState *machine)
object_property_set_bool(cpuobj, "pmu", false, NULL);
}
+if (vmc->no_spe && object_property_find(cpuobj, "spe", NULL)) {
+object_property_set_bool(cpuobj, "spe", false, NULL);
+}
+
if (object_property_find(cpuobj, "reset-cbar", NULL)) {
object_property_set_int(cpuobj, "reset-cbar",
vms->memmap[VIRT_CPUPERIPHS].base,
@@ -2553,8 +2557,11 @@ DEFINE_VIRT_MACHINE_AS_LATEST(5, 2)
static void virt_machine_5_1_options(MachineClass *mc)
{
+VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc));
+
virt_machine_5_2_options(mc);
compat_props_add(mc->compat_props, hw_compat_5_1, hw_compat_5_1_len);
+vmc->no_spe = true;
}
DEFINE_VIRT_MACHINE(5, 1)
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index dff67e1bef..72c269aaa5 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -126,6 +126,7 @@ typedef struct {
bool no_ged; /* Machines < 4.2 has no support for ACPI GED device */
bool kvm_no_adjvtime;
bool acpi_expose_flash;
+bool no_spe;
} VirtMachineClass;
typedef struct {
--
2.17.1
[PATCH v2 08/12] hw/arm/virt: Move kvm pmu setup to virt_cpu_post_init
From: Andrew Jones
Move the KVM PMU setup part of fdt_add_pmu_nodes() to
virt_cpu_post_init(), which is a more appropriate location. Now
fdt_add_pmu_nodes() is also named more appropriately, because it
no longer does anything but fdt node creation.
No functional change intended.
Signed-off-by: Andrew Jones
---
hw/arm/virt.c | 34 ++
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 2ffcb073af..6bacfb668d 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -521,21 +521,12 @@ static void fdt_add_gic_node(VirtMachineState *vms)
static void fdt_add_pmu_nodes(const VirtMachineState *vms)
{
-CPUState *cpu;
-ARMCPU *armcpu;
+ARMCPU *armcpu = ARM_CPU(first_cpu);
uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
-CPU_FOREACH(cpu) {
-armcpu = ARM_CPU(cpu);
-if (!arm_feature(>env, ARM_FEATURE_PMU)) {
-return;
-}
-if (kvm_enabled()) {
-if (kvm_irqchip_in_kernel()) {
-kvm_arm_pmu_set_irq(cpu, PPI(VIRTUAL_PMU_IRQ));
-}
-kvm_arm_pmu_init(cpu);
-}
+if (!arm_feature(>env, ARM_FEATURE_PMU)) {
+assert(!object_property_get_bool(OBJECT(armcpu), "pmu", NULL));
+return;
}
if (vms->gic_version == VIRT_GIC_VERSION_2) {
@@ -544,7 +535,6 @@ static void fdt_add_pmu_nodes(const VirtMachineState *vms)
(1 << vms->smp_cpus) - 1);
}
-armcpu = ARM_CPU(qemu_get_cpu(0));
qemu_fdt_add_subnode(vms->fdt, "/pmu");
if (arm_feature(>env, ARM_FEATURE_V8)) {
const char compat[] = "arm,armv8-pmuv3";
@@ -1674,11 +1664,23 @@ static void finalize_gic_version(VirtMachineState *vms)
static void virt_cpu_post_init(VirtMachineState *vms)
{
-bool aarch64;
+bool aarch64, pmu;
+CPUState *cpu;
aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL);
+pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL);
-if (!kvm_enabled()) {
+if (kvm_enabled()) {
+CPU_FOREACH(cpu) {
+if (pmu) {
+assert(arm_feature(_CPU(cpu)->env, ARM_FEATURE_PMU));
+if (kvm_irqchip_in_kernel()) {
+kvm_arm_pmu_set_irq(cpu, PPI(VIRTUAL_PMU_IRQ));
+}
+kvm_arm_pmu_init(cpu);
+}
+}
+} else {
if (aarch64 && vms->highmem) {
int requested_pa_size = 64 - clz64(vms->highest_gpa);
int pamax = arm_pamax(ARM_CPU(first_cpu));
--
2.17.1
[PATCH v2 12/12] target/arm: spe: Add corresponding doc and test.
Signed-off-by: Haibo Xu
---
docs/system/arm/cpu-features.rst | 20
target/arm/monitor.c | 2 +-
tests/qtest/arm-cpu-features.c | 9 +
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/docs/system/arm/cpu-features.rst b/docs/system/arm/cpu-features.rst
index 2d5c06cd01..5b81b9a560 100644
--- a/docs/system/arm/cpu-features.rst
+++ b/docs/system/arm/cpu-features.rst
@@ -344,3 +344,23 @@ verbose command lines. However, the recommended way to
select vector
lengths is to explicitly enable each desired length. Therefore only
example's (1), (4), and (6) exhibit recommended uses of the properties.
+SPE CPU Property
+==
+
+The SPE CPU property `spe` is used to enable or disable the SPE feature,
+just as the `pmu` CPU property completely enables or disables the PMU.
+
+Currently, this property is only available with KVM mode, and is enabled
+by default if KVM support it. When KVM is enabled, if the host does not
+support SPE, then an error is generated when attempting to enable it.
+
+Following are 2 examples to use this property:
+
+ 1) Disable SPE::
+
+ $ qemu-system-aarch64 -M virt,accel=kvm -cpu max,spe=off
+
+ 2) Implicitly enable it with the `host` CPU type if host cpu
+ support it::
+
+ $ qemu-system-aarch64 -M virt,accel=kvm -cpu host
diff --git a/target/arm/monitor.c b/target/arm/monitor.c
index ba6e01abd0..1b8f08988a 100644
--- a/target/arm/monitor.c
+++ b/target/arm/monitor.c
@@ -99,7 +99,7 @@ QEMU_BUILD_BUG_ON(ARM_MAX_VQ > 16);
* then the order that considers those dependencies must be used.
*/
static const char *cpu_model_advertised_features[] = {
-"aarch64", "pmu", "sve",
+"aarch64", "pmu", "spe", "sve",
"sve128", "sve256", "sve384", "sve512",
"sve640", "sve768", "sve896", "sve1024", "sve1152", "sve1280",
"sve1408", "sve1536", "sve1664", "sve1792", "sve1920", "sve2048",
diff --git a/tests/qtest/arm-cpu-features.c b/tests/qtest/arm-cpu-features.c
index 77b5e30a9c..4d393fb2e2 100644
--- a/tests/qtest/arm-cpu-features.c
+++ b/tests/qtest/arm-cpu-features.c
@@ -494,6 +494,7 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
if (g_str_equal(qtest_get_arch(), "aarch64")) {
bool kvm_supports_sve;
+bool kvm_supports_spe;
char max_name[8], name[8];
uint32_t max_vq, vq;
uint64_t vls;
@@ -512,8 +513,10 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
"with KVM on this host", NULL);
assert_has_feature(qts, "host", "sve");
+assert_has_feature(qts, "host", "spe");
resp = do_query_no_props(qts, "host");
kvm_supports_sve = resp_get_feature(resp, "sve");
+kvm_supports_spe = resp_get_feature(resp, "spe");
vls = resp_get_sve_vls(resp);
qobject_unref(resp);
@@ -573,10 +576,16 @@ static void test_query_cpu_model_expansion_kvm(const void
*data)
} else {
g_assert(vls == 0);
}
+
+if (kvm_supports_spe) {
+assert_set_feature(qts, "host", "spe", false);
+assert_set_feature(qts, "host", "spe", true);
+}
} else {
assert_has_not_feature(qts, "host", "aarch64");
assert_has_not_feature(qts, "host", "pmu");
assert_has_not_feature(qts, "host", "sve");
+assert_has_not_feature(qts, "host", "spe");
}
qtest_quit(qts);
--
2.17.1
[PATCH v2 02/12] target/arm/kvm: spe: Add helper to detect SPE when using KVM
Reviewed-by: Richard Henderson
Signed-off-by: Haibo Xu
---
target/arm/kvm.c | 5 +
target/arm/kvm_arm.h | 13 +
2 files changed, 18 insertions(+)
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 8bb7318378..58f991e890 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -214,6 +214,11 @@ bool kvm_arm_pmu_supported(void)
return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3);
}
+bool kvm_arm_spe_supported(void)
+{
+return kvm_check_extension(kvm_state, KVM_CAP_ARM_SPE_V1);
+}
+
int kvm_arm_get_max_vm_ipa_size(MachineState *ms)
{
KVMState *s = KVM_STATE(ms->accelerator);
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index adb38514bf..f79655674e 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -283,6 +283,14 @@ bool kvm_arm_aarch32_supported(void);
*/
bool kvm_arm_pmu_supported(void);
+/**
+ * kvm_arm_spe_supported:
+ *
+ * Returns: true if the KVM VCPU can enable its SPE
+ * and false otherwise.
+ */
+bool kvm_arm_spe_supported(void);
+
/**
* kvm_arm_sve_supported:
*
@@ -366,6 +374,11 @@ static inline bool kvm_arm_pmu_supported(void)
return false;
}
+static inline bool kvm_arm_spe_supported(void)
+{
+return false;
+}
+
static inline bool kvm_arm_sve_supported(void)
{
return false;
--
2.17.1
[PATCH v2 03/12] target/arm/cpu: spe: Add an option to turn on/off vSPE support
Adds a spe=[on/off] option to enable/disable vSPE support in
guest vCPU.
Signed-off-by: Haibo Xu
---
target/arm/cpu.c | 6 ++
target/arm/cpu.h | 13
target/arm/cpu64.c | 52 ++
3 files changed, 71 insertions(+)
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index c179e0752d..f211958eaa 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1310,6 +1310,12 @@ void arm_cpu_finalize_features(ARMCPU *cpu, Error **errp)
error_propagate(errp, local_err);
return;
}
+
+arm_cpu_spe_finalize(cpu, _err);
+if (local_err != NULL) {
+error_propagate(errp, local_err);
+return;
+}
}
}
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index a1c7d8ebae..baf2bbcee8 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -24,6 +24,7 @@
#include "hw/registerfields.h"
#include "cpu-qom.h"
#include "exec/cpu-defs.h"
+#include "qapi/qapi-types-common.h"
/* ARM processors have a weak memory model */
#define TCG_GUEST_DEFAULT_MO (0)
@@ -196,9 +197,11 @@ typedef struct {
#ifdef TARGET_AARCH64
# define ARM_MAX_VQ16
void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp);
+void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp);
#else
# define ARM_MAX_VQ1
static inline void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) { }
+static inline void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp) { }
#endif
typedef struct ARMVectorReg {
@@ -829,6 +832,8 @@ struct ARMCPU {
bool has_el3;
/* CPU has PMU (Performance Monitor Unit) */
bool has_pmu;
+/* CPU has SPE (Statistical Profiling Extension) */
+OnOffAuto has_spe;
/* CPU has VFP */
bool has_vfp;
/* CPU has Neon */
@@ -3869,6 +3874,14 @@ static inline bool isar_feature_aa64_pmu_8_4(const
ARMISARegisters *id)
FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) != 0xf;
}
+/*
+ * Currently we don't differentiate between the ARMv8.2-SPE and ARMv8.3-SPE.
+ */
+static inline bool isar_feature_aa64_spe(const ARMISARegisters *id)
+{
+return FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMSVER) != 0;
+}
+
static inline bool isar_feature_aa64_rcpc_8_3(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, LRCPC) != 0;
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 3c2b3d9599..4997c4a3c0 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -572,6 +572,55 @@ void aarch64_add_sve_properties(Object *obj)
}
}
+void arm_cpu_spe_finalize(ARMCPU *cpu, Error **errp)
+{
+uint64_t t;
+uint32_t value = 0;
+
+if (cpu->has_spe == ON_OFF_AUTO_AUTO) {
+if (kvm_enabled() && kvm_arm_spe_supported()) {
+cpu->has_spe = ON_OFF_AUTO_ON;
+} else {
+cpu->has_spe = ON_OFF_AUTO_OFF;
+}
+} else if (cpu->has_spe == ON_OFF_AUTO_ON) {
+if (!kvm_enabled() || !kvm_arm_spe_supported()) {
+error_setg(errp, "'spe' cannot be enabled on this host");
+return;
+}
+}
+
+/*
+ * According to the ARM ARM, the ID_AA64DFR0[PMSVER] currently
+ * support 3 values:
+ *
+ * 0b: SPE not implemented
+ * 0b0001: ARMv8.2-SPE implemented
+ * 0b0010: ARMv8.3-SPE implemented
+ *
+ * But the kernel KVM API didn't expose all these 3 values, and
+ * we can only get whether the SPE feature is supported or not.
+ * So here we just set the PMSVER to 1 if this feature was supported.
+ */
+if (cpu->has_spe == ON_OFF_AUTO_ON) {
+value = 1;
+}
+
+t = cpu->isar.id_aa64dfr0;
+t = FIELD_DP64(t, ID_AA64DFR0, PMSVER, value);
+cpu->isar.id_aa64dfr0 = t;
+}
+
+static bool arm_spe_get(Object *obj, Error **errp)
+{
+return ARM_CPU(obj)->has_spe != ON_OFF_AUTO_OFF;
+}
+
+static void arm_spe_set(Object *obj, bool value, Error **errp)
+{
+ARM_CPU(obj)->has_spe = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
+}
+
/* -cpu max: if KVM is enabled, like -cpu host (best possible with this host);
* otherwise, a CPU with as many features enabled as our emulation supports.
* The version of '-cpu max' for qemu-system-arm is defined in cpu.c;
@@ -721,6 +770,9 @@ static void aarch64_max_initfn(Object *obj)
aarch64_add_sve_properties(obj);
object_property_add(obj, "sve-max-vq", "uint32", cpu_max_get_sve_max_vq,
cpu_max_set_sve_max_vq, NULL, NULL);
+
+cpu->has_spe = ON_OFF_AUTO_AUTO;
+object_property_add_bool(obj, "spe", arm_spe_get, arm_spe_set);
}
static const ARMCPUInfo aarch64_cpus[] = {
--
2.17.1
[PATCH v2 00/12] target/arm: Add vSPE support to KVM guest
v2: - Rebased on Andrew's patches[3] - Added compat codes to enable vSPE only for the 5.2 and later machine types [Andrew] - Changed to use the ID register bit to identify the spe feature [Andrew/Richard] - Added the missing field in AcpiMadtGenericCpuInterface definition [Auger] - Split the un-tested userspace irqchip support for vSPE into a separate patch [Andrew] - Added doc and qtest for vSPE [Andrew] Many thanks to Andrew, Richard, Philippe and Auger for their comments. This series add support for SPE(Statistical Profiling Extension)[1] in KVM guest. It's based on Andrew Murray's kernel KVM patches V2[2], and has been tested to ensure that guest can use SPE with valid data. E.g. In host: $ ./qemu-system-aarch64 \ -cpu host -M virt,accel=kvm,gic-version=3 -nographic -m 2048M \ -kernel ./Image-new \ -initrd /boot/initrd.img-5.6.0-rc2+ \ -append "root=/dev/vda rw console=ttyAMA0" -nodefaults -serial stdio\ -drive if=none,file=./xenial.rootfs.ext4,id=hd0,format=raw \ -device virtio-blk-device,drive=hd0 \ In guest: $ perf record -e arm_spe/ts_enable=1,pa_enable=1,pct_enable=1/ \ dd if=/dev/zero of=/dev/null count=1000 $ perf report --dump-raw-trace > spe_buf.txt The spe_buf.txt should contain similar data as below: . ... ARM SPE data: size 135944 bytes . : b0 f4 d3 29 10 00 80 ff a0 PC 0xff80001029d3f4 el1 ns=1 . 0009: 99 0b 00LAT 11 ISSUE . 000c: 98 0d 00LAT 13 TOT . 000f: 52 16 00EV RETIRED L1D-ACCESS TLB-ACCESS . 0012: 49 00 LD . 0014: b2 d0 40 d8 70 00 00 ff 00 VA 0xff70d840d0 . 001d: 9a 01 00LAT 1 XLAT . 0020: 00 00 00PAD . 0023: 71 a5 1f b3 20 14 00 00 00 TS 86447955877 . 002c: b0 7c f9 29 10 00 80 ff a0 PC 0xff80001029f97c el1 ns=1 . 0035: 99 02 00LAT 2 ISSUE . 0038: 98 03 00LAT 3 TOT . 003b: 52 02 00EV RETIRED . 003e: 48 00 INSN-OTHER . 0040: 00 00 00PAD . 0043: 71 ef 1f b3 20 14 00 00 00 TS 86447955951 . 004c: b0 f0 e9 29 10 00 80 ff a0 PC 0xff80001029e9f0 el1 ns=1 . 0055: 99 02 00LAT 2 ISSUE . 0058: 98 03 00LAT 3 TOT . 005b: 52 02 00EV RETIRED If you want to disable the vSPE support, you can use the 'spe=off' cpu property: ./qemu-system-aarch64 \ -cpu host,spe=off -M virt,accel=kvm,gic-version=3 -nographic -m 2048M \ -kernel ./Image-new \ -initrd /boot/initrd.img-5.6.0-rc2+ \ -append "root=/dev/vda rw console=ttyAMA0" -nodefaults -serial stdio\ -drive if=none,file=./xenial.rootfs.ext4,id=hd0,format=raw \ -device virtio-blk-device,drive=hd0 \ Note: (1) Since the kernel patches are still under review, some of the macros in the header files may be changed after merging. We may need to update them accordingly. To be specific, if you want to have a try on this patch series, you needs to replace on the kernel 5.5-rc2 based series, and do minor changes: -#define KVM_CAP_ARM_SPE_V1 179 +#define KVM_CAP_ARM_SPE_V1 184 (2) These patches only add vSPE support in KVM mode, for TCG mode, I'm not sure whether we need to support it. (3) Just followed the 'sve' property, we only allow this feature to be removed from CPUs which enable it by default when the host cpu support it. [1]https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/ posts/statistical-profiling-extension-for-armv8-a [2]https://www.spinics.net/lists/arm-kernel/msg776228.html [3]https://www.mail-archive.com/qemu-devel@nongnu.org/msg727588.html Haibo Xu (12): update Linux headers with new vSPE macros target/arm/kvm: spe: Add helper to detect SPE when using KVM target/arm/cpu: spe: Add an option to turn on/off vSPE support target/arm: spe: Only enable SPE from 5.2 compat machines. target/arm/kvm: spe: Unify device attr operation helper target/arm/kvm: spe: Add device init and set_irq operations hw/arm/virt: Move post cpu realize check into its own function hw/arm/virt: Move kvm pmu setup to virt_cpu_post_init hw/arm/virt: spe: Add SPE fdt binding for virt machine target/arm/cpu: spe: Enable spe to work with host cpu target/arm/kvm: spe: Enable userspace irqchip support. target/arm: spe
[PATCH v2 10/12] target/arm/cpu: spe: Enable spe to work with host cpu
Turn on the spe cpu property by default if host cpu
support it, i.e. we can now do '-cpu max|host' to add
the vSPE, and '-cpu max|host,spe=off' to remove it.
Signed-off-by: Haibo Xu
---
target/arm/cpu.c | 3 +++
target/arm/cpu.h | 2 ++
target/arm/cpu64.c | 7 ++-
target/arm/kvm64.c | 12
4 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 786cc6134c..58f12d6eb5 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -2271,6 +2271,9 @@ static void arm_host_initfn(Object *obj)
kvm_arm_set_cpu_features_from_host(cpu);
if (arm_feature(>env, ARM_FEATURE_AARCH64)) {
aarch64_add_sve_properties(obj);
+
+cpu->has_spe = ON_OFF_AUTO_AUTO;
+aarch64_add_spe_properties(obj);
}
arm_cpu_post_init(obj);
}
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 395a1e5df8..5a3ea876c8 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -1040,6 +1040,7 @@ void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq);
void aarch64_sve_change_el(CPUARMState *env, int old_el,
int new_el, bool el0_a64);
void aarch64_add_sve_properties(Object *obj);
+void aarch64_add_spe_properties(Object *obj);
/*
* SVE registers are encoded in KVM's memory in an endianness-invariant format.
@@ -1071,6 +1072,7 @@ static inline void aarch64_sve_change_el(CPUARMState
*env, int o,
int n, bool a)
{ }
static inline void aarch64_add_sve_properties(Object *obj) { }
+static inline void aarch64_add_spe_properties(Object *obj) { }
#endif
#if !defined(CONFIG_TCG)
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 4997c4a3c0..d38c55e2ca 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -621,6 +621,11 @@ static void arm_spe_set(Object *obj, bool value, Error
**errp)
ARM_CPU(obj)->has_spe = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
}
+void aarch64_add_spe_properties(Object *obj)
+{
+object_property_add_bool(obj, "spe", arm_spe_get, arm_spe_set);
+}
+
/* -cpu max: if KVM is enabled, like -cpu host (best possible with this host);
* otherwise, a CPU with as many features enabled as our emulation supports.
* The version of '-cpu max' for qemu-system-arm is defined in cpu.c;
@@ -772,7 +777,7 @@ static void aarch64_max_initfn(Object *obj)
cpu_max_set_sve_max_vq, NULL, NULL);
cpu->has_spe = ON_OFF_AUTO_AUTO;
-object_property_add_bool(obj, "spe", arm_spe_get, arm_spe_set);
+aarch64_add_spe_properties(obj);
}
static const ARMCPUInfo aarch64_cpus[] = {
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 5a2032fc9e..3f0a09c05b 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -515,6 +515,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
*/
int fdarray[3];
bool sve_supported;
+bool spe_supported;
uint64_t features = 0;
uint64_t t;
int err;
@@ -655,6 +656,8 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
}
sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) >
0;
+spe_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION,
+ KVM_CAP_ARM_SPE_V1) > 0;
kvm_arm_destroy_scratch_host_vcpu(fdarray);
@@ -668,6 +671,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures
*ahcf)
t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
ahcf->isar.id_aa64pfr0 = t;
}
+if (!spe_supported) {
+t = ahcf->isar.id_aa64dfr0;
+t = FIELD_DP64(t, ID_AA64DFR0, PMSVER, 0);
+ahcf->isar.id_aa64dfr0 = t;
+}
/*
* We can assume any KVM supporting CPU is at least a v8
@@ -830,6 +838,10 @@ int kvm_arch_init_vcpu(CPUState *cs)
assert(kvm_arm_sve_supported());
cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
}
+if (cpu_isar_feature(aa64_spe, cpu)) {
+assert(kvm_arm_spe_supported());
+cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SPE_V1;
+}
/* Do KVM_ARM_VCPU_INIT ioctl */
ret = kvm_arm_vcpu_init(cs);
--
2.17.1
[PATCH v2 09/12] hw/arm/virt: spe: Add SPE fdt binding for virt machine
Add a virtual SPE device for virt machine while using
PPI 5 for SPE overflow interrupt number which has already
selected in kvmtool.
Signed-off-by: Haibo Xu
---
hw/arm/virt-acpi-build.c| 3 +++
hw/arm/virt.c | 43 -
include/hw/acpi/acpi-defs.h | 3 +++
include/hw/arm/virt.h | 1 +
target/arm/cpu.c| 2 ++
target/arm/cpu.h| 2 ++
6 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
index 9efd7a3881..3fd80fda53 100644
--- a/hw/arm/virt-acpi-build.c
+++ b/hw/arm/virt-acpi-build.c
@@ -665,6 +665,9 @@ build_madt(GArray *table_data, BIOSLinker *linker,
VirtMachineState *vms)
if (arm_feature(>env, ARM_FEATURE_PMU)) {
gicc->performance_interrupt = cpu_to_le32(PPI(VIRTUAL_PMU_IRQ));
}
+if (cpu_isar_feature(aa64_spe, armcpu)) {
+gicc->spe_interrupt = cpu_to_le32(PPI(VIRTUAL_SPE_IRQ));
+}
if (vms->virt) {
gicc->vgic_interrupt = cpu_to_le32(PPI(ARCH_GIC_MAINT_IRQ));
}
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 6bacfb668d..bdb1ce925c 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -545,6 +545,32 @@ static void fdt_add_pmu_nodes(const VirtMachineState *vms)
}
}
+static void fdt_add_spe_nodes(const VirtMachineState *vms)
+{
+ARMCPU *armcpu = ARM_CPU(first_cpu);
+uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
+
+if (!cpu_isar_feature(aa64_spe, armcpu)) {
+assert(!object_property_get_bool(OBJECT(armcpu), "spe", NULL));
+return;
+}
+
+if (vms->gic_version == VIRT_GIC_VERSION_2) {
+irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
+ GIC_FDT_IRQ_PPI_CPU_WIDTH,
+ (1 << vms->smp_cpus) - 1);
+}
+
+qemu_fdt_add_subnode(vms->fdt, "/spe");
+if (arm_feature(>env, ARM_FEATURE_V8)) {
+const char compat[] = "arm,statistical-profiling-extension-v1";
+qemu_fdt_setprop(vms->fdt, "/spe", "compatible",
+ compat, sizeof(compat));
+qemu_fdt_setprop_cells(vms->fdt, "/spe", "interrupts",
+ GIC_FDT_IRQ_TYPE_PPI, VIRTUAL_SPE_IRQ,
irqflags);
+}
+}
+
static inline DeviceState *create_acpi_ged(VirtMachineState *vms)
{
DeviceState *dev;
@@ -717,6 +743,10 @@ static void create_gic(VirtMachineState *vms)
qdev_get_gpio_in(vms->gic, ppibase
+ VIRTUAL_PMU_IRQ));
+qdev_connect_gpio_out_named(cpudev, "spe-interrupt", 0,
+qdev_get_gpio_in(vms->gic, ppibase
+ + VIRTUAL_SPE_IRQ));
+
sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev,
ARM_CPU_IRQ));
sysbus_connect_irq(gicbusdev, i + smp_cpus,
qdev_get_gpio_in(cpudev, ARM_CPU_FIQ));
@@ -1664,11 +1694,12 @@ static void finalize_gic_version(VirtMachineState *vms)
static void virt_cpu_post_init(VirtMachineState *vms)
{
-bool aarch64, pmu;
+bool aarch64, pmu, spe;
CPUState *cpu;
aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL);
pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL);
+spe = object_property_get_bool(OBJECT(first_cpu), "spe", NULL);
if (kvm_enabled()) {
CPU_FOREACH(cpu) {
@@ -1679,6 +1710,14 @@ static void virt_cpu_post_init(VirtMachineState *vms)
}
kvm_arm_pmu_init(cpu);
}
+
+if (spe) {
+assert(ARM_CPU(cpu)->has_spe == ON_OFF_AUTO_ON);
+if (kvm_irqchip_in_kernel()) {
+kvm_arm_spe_set_irq(cpu, PPI(VIRTUAL_SPE_IRQ));
+}
+kvm_arm_spe_init(cpu);
+}
}
} else {
if (aarch64 && vms->highmem) {
@@ -1927,6 +1966,8 @@ static void machvirt_init(MachineState *machine)
fdt_add_pmu_nodes(vms);
+fdt_add_spe_nodes(vms);
+
create_uart(vms, VIRT_UART, sysmem, serial_hd(0));
if (vms->secure) {
diff --git a/include/hw/acpi/acpi-defs.h b/include/hw/acpi/acpi-defs.h
index 38a42f409a..21e58f27c5 100644
--- a/include/hw/acpi/acpi-defs.h
+++ b/include/hw/acpi/acpi-defs.h
@@ -302,6 +302,9 @@ struct AcpiMadtGenericCpuInterface {
uint32_t vgic_interrupt;
uint64_t gicr_base_address;
uint64_t arm_mpidr;
+uint8_t efficiency_class;
+uint8_t reserved2[1];
+uint16_t spe_interrupt; /* ACPI 6.3 */
} QEMU_PACKED;
typedef struct AcpiMadtGenericCpuInterface AcpiMadtGenericCpuInterface;
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index 72c269aaa5
[PATCH v2 07/12] hw/arm/virt: Move post cpu realize check into its own function
From: Andrew Jones
We'll add more to this new function in coming patches so we also
state the gic must be created and call it below create_gic().
No functional change intended.
Signed-off-by: Andrew Jones
---
hw/arm/virt.c | 38 ++
1 file changed, 22 insertions(+), 16 deletions(-)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 3f6d26c531..2ffcb073af 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -1672,6 +1672,26 @@ static void finalize_gic_version(VirtMachineState *vms)
}
}
+static void virt_cpu_post_init(VirtMachineState *vms)
+{
+bool aarch64;
+
+aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL);
+
+if (!kvm_enabled()) {
+if (aarch64 && vms->highmem) {
+int requested_pa_size = 64 - clz64(vms->highest_gpa);
+int pamax = arm_pamax(ARM_CPU(first_cpu));
+
+if (pamax < requested_pa_size) {
+error_report("VCPU supports less PA bits (%d) than requested "
+"by the memory map (%d)", pamax,
requested_pa_size);
+exit(1);
+}
+}
+ }
+}
+
static void machvirt_init(MachineState *machine)
{
VirtMachineState *vms = VIRT_MACHINE(machine);
@@ -1890,22 +1910,6 @@ static void machvirt_init(MachineState *machine)
fdt_add_timer_nodes(vms);
fdt_add_cpu_nodes(vms);
- if (!kvm_enabled()) {
-ARMCPU *cpu = ARM_CPU(first_cpu);
-bool aarch64 = object_property_get_bool(OBJECT(cpu), "aarch64", NULL);
-
-if (aarch64 && vms->highmem) {
-int requested_pa_size, pamax = arm_pamax(cpu);
-
-requested_pa_size = 64 - clz64(vms->highest_gpa);
-if (pamax < requested_pa_size) {
-error_report("VCPU supports less PA bits (%d) than requested "
-"by the memory map (%d)", pamax,
requested_pa_size);
-exit(1);
-}
-}
-}
-
memory_region_add_subregion(sysmem, vms->memmap[VIRT_MEM].base,
machine->ram);
if (machine->device_memory) {
@@ -1917,6 +1921,8 @@ static void machvirt_init(MachineState *machine)
create_gic(vms);
+virt_cpu_post_init(vms);
+
fdt_add_pmu_nodes(vms);
create_uart(vms, VIRT_UART, sysmem, serial_hd(0));
--
2.17.1
[PATCH v2 05/12] target/arm/kvm: spe: Unify device attr operation helper
From: Andrew Jones
Rename kvm_arm_pmu_set_attr() to kvm_arm_set_device_attr(),
So both the vPMU and vSPE device can share the same API.
Signed-off-by: Andrew Jones
---
target/arm/kvm64.c | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index ef1e960285..8ffd31ffdf 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -397,19 +397,20 @@ static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu,
target_ulong addr)
return NULL;
}
-static bool kvm_arm_pmu_set_attr(CPUState *cs, struct kvm_device_attr *attr)
+static bool kvm_arm_set_device_attr(CPUState *cs, struct kvm_device_attr *attr,
+const char *name)
{
int err;
err = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr);
if (err != 0) {
-error_report("PMU: KVM_HAS_DEVICE_ATTR: %s", strerror(-err));
+error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err));
return false;
}
err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, attr);
if (err != 0) {
-error_report("PMU: KVM_SET_DEVICE_ATTR: %s", strerror(-err));
+error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err));
return false;
}
@@ -426,7 +427,7 @@ void kvm_arm_pmu_init(CPUState *cs)
if (!ARM_CPU(cs)->has_pmu) {
return;
}
-if (!kvm_arm_pmu_set_attr(cs, )) {
+if (!kvm_arm_set_device_attr(cs, , "PMU")) {
error_report("failed to init PMU");
abort();
}
@@ -443,7 +444,7 @@ void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
if (!ARM_CPU(cs)->has_pmu) {
return;
}
-if (!kvm_arm_pmu_set_attr(cs, )) {
+if (!kvm_arm_set_device_attr(cs, , "PMU")) {
error_report("failed to set irq for PMU");
abort();
}
--
2.17.1
[PATCH v2 01/12] update Linux headers with new vSPE macros
Signed-off-by: Haibo Xu
---
linux-headers/asm-arm64/kvm.h | 4
linux-headers/linux/kvm.h | 1 +
2 files changed, 5 insertions(+)
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index 9e34f0f875..802319ee02 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -106,6 +106,7 @@ struct kvm_regs {
#define KVM_ARM_VCPU_SVE 4 /* enable SVE for this CPU */
#define KVM_ARM_VCPU_PTRAUTH_ADDRESS 5 /* VCPU uses address authentication */
#define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */
+#define KVM_ARM_VCPU_SPE_V17 /* Support guest SPEv1 */
struct kvm_vcpu_init {
__u32 target;
@@ -334,6 +335,9 @@ struct kvm_vcpu_events {
#define KVM_ARM_VCPU_TIMER_IRQ_PTIMER1
#define KVM_ARM_VCPU_PVTIME_CTRL 2
#define KVM_ARM_VCPU_PVTIME_IPA 0
+#define KVM_ARM_VCPU_SPE_V1_CTRL 3
+#define KVM_ARM_VCPU_SPE_V1_IRQ 0
+#define KVM_ARM_VCPU_SPE_V1_INIT 1
/* KVM_IRQ_LINE irq field index values */
#define KVM_ARM_IRQ_VCPU2_SHIFT28
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index a28c366737..8840cbb01c 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1031,6 +1031,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_PPC_SECURE_GUEST 181
#define KVM_CAP_HALT_POLL 182
#define KVM_CAP_ASYNC_PF_INT 183
+#define KVM_CAP_ARM_SPE_V1 184
#ifdef KVM_CAP_IRQ_ROUTING
--
2.17.1
Re: [PATCH 0/7] target/arm: Add vSPE support to KVM guest
On Mon, 31 Aug 2020 at 15:56, Auger Eric wrote: > > Hi Haibo, > > On 8/7/20 10:10 AM, Haibo Xu wrote: > > This series add support for SPE(Statistical Profiling Extension)[1] > > in KVM guest. It's based on Andrew Murray's kernel KVM patches V2[2], > > and has been tested to ensure that guest can use SPE with valid data. > > E.g. > > > > In host: > > $ ./qemu-system-aarch64 \ > > -cpu host -M virt,accel=kvm,gic-version=3 -nographic -m 2048M \ > > -kernel ./Image-new \ > > -initrd /boot/initrd.img-5.6.0-rc2+ \ > > -append "root=/dev/vda rw console=ttyAMA0" -nodefaults -serial > > stdio\ > > -drive if=none,file=./xenial.rootfs.ext4,id=hd0,format=raw \ > > -device virtio-blk-device,drive=hd0 \ > > > > In guest: > > $ perf record -e arm_spe/ts_enable=1,pa_enable=1,pct_enable=1/ \ > > dd if=/dev/zero of=/dev/null count=1000 > > $ perf report --dump-raw-trace > spe_buf.txt > > > > The spe_buf.txt should contain similar data as below: > > > > . ... ARM SPE data: size 135944 bytes > > . : b0 f4 d3 29 10 00 80 ff a0 PC > > 0xff80001029d3f4 el1 ns=1 > > . 0009: 99 0b 00LAT 11 ISSUE > > . 000c: 98 0d 00LAT 13 TOT > > . 000f: 52 16 00EV RETIRED > > L1D-ACCESS TLB-ACCESS > > . 0012: 49 00 LD > > . 0014: b2 d0 40 d8 70 00 00 ff 00 VA > > 0xff70d840d0 > > . 001d: 9a 01 00LAT 1 XLAT > > . 0020: 00 00 00PAD > > . 0023: 71 a5 1f b3 20 14 00 00 00 TS 86447955877 > > . 002c: b0 7c f9 29 10 00 80 ff a0 PC > > 0xff80001029f97c el1 ns=1 > > . 0035: 99 02 00LAT 2 ISSUE > > . 0038: 98 03 00LAT 3 TOT > > . 003b: 52 02 00EV RETIRED > > . 003e: 48 00 INSN-OTHER > > . 0040: 00 00 00PAD > > . 0043: 71 ef 1f b3 20 14 00 00 00 TS 86447955951 > > . 004c: b0 f0 e9 29 10 00 80 ff a0 PC > > 0xff80001029e9f0 el1 ns=1 > > . 0055: 99 02 00LAT 2 ISSUE > > . 0058: 98 03 00LAT 3 TOT > > . 005b: 52 02 00EV RETIRED > > > > If you want to disable the vSPE support, you can use the 'spe=off' cpu > > property: > > > > ./qemu-system-aarch64 \ > > -cpu host,spe=off -M virt,accel=kvm,gic-version=3 -nographic -m > > 2048M \ > > -kernel ./Image-new \ > > -initrd /boot/initrd.img-5.6.0-rc2+ \ > > -append "root=/dev/vda rw console=ttyAMA0" -nodefaults -serial > > stdio\ > > -drive if=none,file=./xenial.rootfs.ext4,id=hd0,format=raw \ > > -device virtio-blk-device,drive=hd0 \ > > > > Note: > > (1) Since the kernel patches are still under review, some of the macros > > in the header files may be changed after merging. We may need to > > update them accordingly. > to be more explicit one needs to replace on the kernel 5.5-rc2 based series > > -#define KVM_CAP_ARM_SPE_V1 179 > +#define KVM_CAP_ARM_SPE_V1 184 > > I got misleaded ;-) > > + Andrew in CC as he contributed the kernel part. > > For information, I have been working on a kvm unit test series for > testing SPE. I will send an RFC, most probably this week. At the moment > I still face some weirdness such as some unexpected Service state in the > syndrome register. Anyway I will share the existing code so that we can > discuss the issues. > > Are there any plans to respin the kernel series > > Thanks > > Eric > Hi Eric, Thanks for elaborating on the above macro definition! The next version of the kernel series are supposed to be sent out in mid-September, and it should not change so much except for some macro definitions. Regards, Haibo > > (2) These patches only add vSPE support in KVM mode, for TCG mode, I'm > > not sure whether we need to support it. > > (3) Just followed the 'pmu' property, we only allow this feature to be > > removed from CPUs which enable
Re: [PATCH 6/7] hw/arm/virt: spe: Add SPE fdt binding for virt machine
On Sat, 29 Aug 2020 at 23:22, Auger Eric wrote:
>
> Hi Haibo,
>
> On 8/7/20 10:10 AM, Haibo Xu wrote:
> > Add a virtual SPE device for virt machine while using PPI
> > 5 for SPE overflow interrupt number.
> >
> > Signed-off-by: Haibo Xu
> > ---
> > hw/arm/virt-acpi-build.c| 3 +++
> > hw/arm/virt.c | 42 +
> > include/hw/acpi/acpi-defs.h | 1 +
> > include/hw/arm/virt.h | 1 +
> > target/arm/cpu.c| 2 ++
> > target/arm/cpu.h| 2 ++
> > target/arm/kvm.c| 6 ++
> > 7 files changed, 57 insertions(+)
> >
> > diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
> > index 91f0df7b13..5073ba22a5 100644
> > --- a/hw/arm/virt-acpi-build.c
> > +++ b/hw/arm/virt-acpi-build.c
> > @@ -666,6 +666,9 @@ build_madt(GArray *table_data, BIOSLinker *linker,
> > VirtMachineState *vms)
> > if (arm_feature(>env, ARM_FEATURE_PMU)) {
> > gicc->performance_interrupt =
> > cpu_to_le32(PPI(VIRTUAL_PMU_IRQ));
> > }
> > +if (arm_feature(>env, ARM_FEATURE_SPE)) {
> > +gicc->spe_interrupt = cpu_to_le32(PPI(VIRTUAL_SPE_IRQ));
> > +}
> > if (vms->virt) {
> > gicc->vgic_interrupt = cpu_to_le32(PPI(ARCH_GIC_MAINT_IRQ));
> > }
> > diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> > index ecfee362a1..c40819705d 100644
> > --- a/hw/arm/virt.c
> > +++ b/hw/arm/virt.c
> > @@ -555,6 +555,42 @@ static void fdt_add_pmu_nodes(const VirtMachineState
> > *vms)
> > }
> > }
> >
> > +static void fdt_add_spe_nodes(const VirtMachineState *vms)
> > +{
> > +CPUState *cpu;
> > +ARMCPU *armcpu;
> > +uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
> > +
> > +CPU_FOREACH(cpu) {
> > +armcpu = ARM_CPU(cpu);
> > +if (!arm_feature(>env, ARM_FEATURE_SPE)) {
> > +return;
> > +}
> > +if (kvm_enabled()) {
> > +if (kvm_irqchip_in_kernel()) {
> > +kvm_arm_spe_set_irq(cpu, PPI(VIRTUAL_SPE_IRQ));
> > +}
> > +kvm_arm_spe_init(cpu);
> > +}
> > +}
> > +
> > +if (vms->gic_version == VIRT_GIC_VERSION_2) {
> > +irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
> > + GIC_FDT_IRQ_PPI_CPU_WIDTH,
> > + (1 << vms->smp_cpus) - 1);
> > +}
> > +
> > +armcpu = ARM_CPU(qemu_get_cpu(0));
> > +qemu_fdt_add_subnode(vms->fdt, "/spe");
> > +if (arm_feature(>env, ARM_FEATURE_V8)) {
> > +const char compat[] = "arm,statistical-profiling-extension-v1";
> > +qemu_fdt_setprop(vms->fdt, "/spe", "compatible",
> > + compat, sizeof(compat));
> > +qemu_fdt_setprop_cells(vms->fdt, "/spe", "interrupts",
> > + GIC_FDT_IRQ_TYPE_PPI, VIRTUAL_SPE_IRQ,
> > irqflags);
> > +}
> > +}
> > +
> > static inline DeviceState *create_acpi_ged(VirtMachineState *vms)
> > {
> > DeviceState *dev;
> > @@ -727,6 +763,10 @@ static void create_gic(VirtMachineState *vms)
> > qdev_get_gpio_in(vms->gic, ppibase
> > + VIRTUAL_PMU_IRQ));
> >
> > +qdev_connect_gpio_out_named(cpudev, "spe-interrupt", 0,
> > +qdev_get_gpio_in(vms->gic, ppibase
> > + + VIRTUAL_SPE_IRQ));
> > +
> > sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev,
> > ARM_CPU_IRQ));
> > sysbus_connect_irq(gicbusdev, i + smp_cpus,
> > qdev_get_gpio_in(cpudev, ARM_CPU_FIQ));
> > @@ -1915,6 +1955,8 @@ static void machvirt_init(MachineState *machine)
> >
> > fdt_add_pmu_nodes(vms);
> >
> > +fdt_add_spe_nodes(vms);
> > +
> > create_uart(vms, VIRT_UART, sysmem, serial_hd(0));
> >
> > if (vms->secure) {
> > diff --git a/include/hw/acpi/acpi-defs.h b/include/hw/acpi/acpi-defs.h
> > index 38a42f409a..56a7f38ae4 100644
> > --- a/include/hw/acpi/acpi-defs.h
> > +++ b/include/hw/acpi/acpi-defs.h
> > @@ -302,6 +302,7 @@ struct AcpiMadtGenericCpuInterface {
>
Re: [PATCH 7/7] target/arm/cpu: spe: Enable spe to work with host cpu
On Wed, 12 Aug 2020 at 00:50, Andrew Jones wrote:
>
> On Tue, Aug 11, 2020 at 11:15:42AM +0800, Haibo Xu wrote:
> > > > +if (!cpu->has_spe || !kvm_enabled()) {
> > > > +unset_feature(env, ARM_FEATURE_SPE);
> > > > +}
> > >
> > > I don't think this should be necessary.
> > >
> >
> > Yes, I have tried to remove this check, and the vSPE can still work
> > correctly.
> > But I don't know whether there are some corner cases that trigger an error.
> > The similar logic is added in commit 929e754d5a to enable vPMU support.
>
> I think the PMU logic needs a cleanup, rather than to be imitated.
>
> >
> > > > +
> > > > if (!arm_feature(env, ARM_FEATURE_EL2)) {
> > > > /* Disable the hypervisor feature bits in the processor feature
> > > > * registers if we don't have EL2. These are id_pfr1[15:12] and
> > > > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> > > > index be045ccc5f..4ea58afc1d 100644
> > > > --- a/target/arm/kvm64.c
> > > > +++ b/target/arm/kvm64.c
> > > > @@ -679,6 +679,7 @@ bool
> > kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
> > > > features |= 1ULL << ARM_FEATURE_AARCH64;
> > > > features |= 1ULL << ARM_FEATURE_PMU;
> > > > features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
> > > > +features |= 1ULL << ARM_FEATURE_SPE;
> > >
> > > No, SPE is not a feature we assume is present in v8.0 CPUs.
> > >
> >
> > Yes, SPE is an optional feature for v8.2. How about changing to the
> > following logic:
> >
> > spe_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SPE_V1)
> > > 0;
> > if (spe_supported) {
> > features |= 1ULL << ARM_FEATURE_SPE;
> > }
>
> Yes, except you need to drop the ARM_FEATURE_SPE define and use the ID
> register bit instead like "sve_supported" does.
>
> >
> > > >
> > > > ahcf->features = features;
> > > >
> > > > @@ -826,6 +827,14 @@ int kvm_arch_init_vcpu(CPUState *cs)
> > > > } else {
> > > > env->features &= ~(1ULL << ARM_FEATURE_PMU);
> > > > }
> > > > +if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_SPE_V1)) {
> > > > +cpu->has_spe = false;
> > > > +}
> > > > +if (cpu->has_spe) {
> > > > +cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SPE_V1;
> > > > +} else {
> > > > +env->features &= ~(1ULL << ARM_FEATURE_SPE);
> > > > +}
> > >
> > > The PMU code above this isn't a good pattern to copy. The SVE code below
> > > is better. SVE uses an ID bit and doesn't do the redundant KVM cap check.
> > > It'd be nice to cleanup the PMU code (with a separate patch) and then add
> > > SPE in a better way.
> > >
> >
> > I noticed that Peter had sent out a mail
> > <https://www.mail-archive.com/qemu-devel@nongnu.org/msg727640.html> to talk
> > about the feature-identification strategy.
> > So shall we adapt it to the vPMU and vSPE feature?
>
> At least SPE. You'll have to double check that it makes sense to do for
> PMU. But, if so, then it should be done with a separate series.
>
Ok, will adapt the SPE support to this new feature-identification
strategy first, and
investigate whether it makes sense to do so for PMU later.
Thank you very much for helping review the patch series!
> Thanks,
> drew
>
Re: [PATCH 6/7] hw/arm/virt: spe: Add SPE fdt binding for virt machine
On Wed, 12 Aug 2020 at 00:40, Andrew Jones wrote:
> On Tue, Aug 11, 2020 at 10:38:02AM +0800, Haibo Xu wrote:
> > On Mon, 10 Aug 2020 at 19:05, Andrew Jones wrote:
> > >
> > > On Fri, Aug 07, 2020 at 08:10:36AM +, Haibo Xu wrote:
> > > > Add a virtual SPE device for virt machine while using PPI
> > > > 5 for SPE overflow interrupt number.
> > >
> > > Any reason PPI 5 was selected?
> > >
> >
> > No special reason to choose PPI 5. Just re-use the setting in kvmtool.
>
> Please write in the commit message that kvmtool has already selected PPI 5
> for this purpose.
>
Ok, will fix it.
> > > > +fdt_add_spe_nodes(vms);
> > > > +
> > >
> > > You didn't add any compat code, which means all virt machine types are
> now
> > > getting an SPE FDT node, ACPI table change, and, most importantly, PPI
> 5
> > > has gone from unallocated to allocated. We definitely need compat code.
> > >
> >
> > So the 'compat code' here means to only add the SPE node in KVM mode?
>
> No, it means only add it for the 5.2 and later machine types. You'll see
> what I mean when you study the patchset I pointed out, which is also only
> for 5.2 and later machine types.
>
Ok, thanks for the clarification!
> > > > +if (switched_level & KVM_ARM_DEV_SPE) {
> > > > +qemu_set_irq(cpu->spe_interrupt,
> > > > + !!(run->s.regs.device_irq_level &
> KVM_ARM_DEV_SPE));
> > > > +switched_level &= ~KVM_ARM_DEV_SPE;
> > > > +}
> > > > +
> > >
> > > Did you test with a userspace irqchip?
> >
> > No, I just tested with an in-kernel irqchip.
> > Actually, the current kernel vSPE patch doesn't support a userspace
> irqchip.
> > AFAIK, the userspace irqchip support should be ready in the next
> > kernel patch series
> > which will be sent out for review in the middle of September.
>
> It probably doesn't hurt to do the above hunk already, hoping it will just
> work when it's possible to test, but I generally prefer only adding tested
> code. Maybe this hunk should be a separate patch with a commit message
> explaining that it's untested?
>
Good idea! I will drop the hunk in this series, and send out a separate
patch to enable it
once the kernel support is ready!
> Thanks,
> drew
>
>
Re: [PATCH 7/7] target/arm/cpu: spe: Enable spe to work with host cpu
On Mon, 10 Aug 2020 at 19:16, Andrew Jones wrote:
>
> On Fri, Aug 07, 2020 at 08:10:37AM +, Haibo Xu wrote:
> > Turn on the spe cpu property by default when working with host
> > cpu type in KVM mode, i.e. we can now do '-cpu host' to add the
> > vSPE, and '-cpu host,spe=off' to remove it.
>
> -cpu max with KVM should also enable it by default
>
Ok, will fix it!
> >
> > Signed-off-by: Haibo Xu
> > ---
> > target/arm/cpu.c | 4
> > target/arm/kvm64.c | 9 +
> > 2 files changed, 13 insertions(+)
> >
> > diff --git a/target/arm/cpu.c b/target/arm/cpu.c
> > index 67ab0089fd..42fa99953c 100644
> > --- a/target/arm/cpu.c
> > +++ b/target/arm/cpu.c
> > @@ -1719,6 +1719,10 @@ static void arm_cpu_realizefn(DeviceState *dev,
Error **errp)
> > cpu->pmceid1 = 0;
> > }
> >
> > +if (!cpu->has_spe || !kvm_enabled()) {
> > +unset_feature(env, ARM_FEATURE_SPE);
> > +}
>
> I don't think this should be necessary.
>
Yes, I have tried to remove this check, and the vSPE can still work
correctly.
But I don't know whether there are some corner cases that trigger an error.
The similar logic is added in commit 929e754d5a to enable vPMU support.
> > +
> > if (!arm_feature(env, ARM_FEATURE_EL2)) {
> > /* Disable the hypervisor feature bits in the processor feature
> > * registers if we don't have EL2. These are id_pfr1[15:12] and
> > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> > index be045ccc5f..4ea58afc1d 100644
> > --- a/target/arm/kvm64.c
> > +++ b/target/arm/kvm64.c
> > @@ -679,6 +679,7 @@ bool
kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
> > features |= 1ULL << ARM_FEATURE_AARCH64;
> > features |= 1ULL << ARM_FEATURE_PMU;
> > features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
> > +features |= 1ULL << ARM_FEATURE_SPE;
>
> No, SPE is not a feature we assume is present in v8.0 CPUs.
>
Yes, SPE is an optional feature for v8.2. How about changing to the
following logic:
spe_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SPE_V1)
> 0;
if (spe_supported) {
features |= 1ULL << ARM_FEATURE_SPE;
}
> >
> > ahcf->features = features;
> >
> > @@ -826,6 +827,14 @@ int kvm_arch_init_vcpu(CPUState *cs)
> > } else {
> > env->features &= ~(1ULL << ARM_FEATURE_PMU);
> > }
> > +if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_SPE_V1)) {
> > +cpu->has_spe = false;
> > +}
> > +if (cpu->has_spe) {
> > +cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SPE_V1;
> > +} else {
> > +env->features &= ~(1ULL << ARM_FEATURE_SPE);
> > +}
>
> The PMU code above this isn't a good pattern to copy. The SVE code below
> is better. SVE uses an ID bit and doesn't do the redundant KVM cap check.
> It'd be nice to cleanup the PMU code (with a separate patch) and then add
> SPE in a better way.
>
I noticed that Peter had sent out a mail
<https://www.mail-archive.com/qemu-devel@nongnu.org/msg727640.html> to talk
about the feature-identification strategy.
So shall we adapt it to the vPMU and vSPE feature?
> > if (cpu_isar_feature(aa64_sve, cpu)) {
> > assert(kvm_arm_sve_supported());
> > cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
> > --
> > 2.17.1
> >
>
> Thanks,
> drew
>
Re: [PATCH 6/7] hw/arm/virt: spe: Add SPE fdt binding for virt machine
On Mon, 10 Aug 2020 at 19:05, Andrew Jones wrote:
>
> On Fri, Aug 07, 2020 at 08:10:36AM +, Haibo Xu wrote:
> > Add a virtual SPE device for virt machine while using PPI
> > 5 for SPE overflow interrupt number.
>
> Any reason PPI 5 was selected?
>
No special reason to choose PPI 5. Just re-use the setting in kvmtool.
> >
> > Signed-off-by: Haibo Xu
> > ---
> > hw/arm/virt-acpi-build.c| 3 +++
> > hw/arm/virt.c | 42 +
> > include/hw/acpi/acpi-defs.h | 1 +
> > include/hw/arm/virt.h | 1 +
> > target/arm/cpu.c| 2 ++
> > target/arm/cpu.h| 2 ++
> > target/arm/kvm.c| 6 ++
> > 7 files changed, 57 insertions(+)
> >
> > diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
> > index 91f0df7b13..5073ba22a5 100644
> > --- a/hw/arm/virt-acpi-build.c
> > +++ b/hw/arm/virt-acpi-build.c
> > @@ -666,6 +666,9 @@ build_madt(GArray *table_data, BIOSLinker *linker,
> > VirtMachineState *vms)
> > if (arm_feature(>env, ARM_FEATURE_PMU)) {
> > gicc->performance_interrupt =
> > cpu_to_le32(PPI(VIRTUAL_PMU_IRQ));
> > }
> > +if (arm_feature(>env, ARM_FEATURE_SPE)) {
> > +gicc->spe_interrupt = cpu_to_le32(PPI(VIRTUAL_SPE_IRQ));
> > +}
> > if (vms->virt) {
> > gicc->vgic_interrupt = cpu_to_le32(PPI(ARCH_GIC_MAINT_IRQ));
> > }
> > diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> > index ecfee362a1..c40819705d 100644
> > --- a/hw/arm/virt.c
> > +++ b/hw/arm/virt.c
> > @@ -555,6 +555,42 @@ static void fdt_add_pmu_nodes(const VirtMachineState
> > *vms)
> > }
> > }
> >
> > +static void fdt_add_spe_nodes(const VirtMachineState *vms)
> > +{
> > +CPUState *cpu;
> > +ARMCPU *armcpu;
> > +uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
> > +
> > +CPU_FOREACH(cpu) {
> > +armcpu = ARM_CPU(cpu);
> > +if (!arm_feature(>env, ARM_FEATURE_SPE)) {
> > +return;
> > +}
> > +if (kvm_enabled()) {
> > +if (kvm_irqchip_in_kernel()) {
> > +kvm_arm_spe_set_irq(cpu, PPI(VIRTUAL_SPE_IRQ));
> > +}
> > +kvm_arm_spe_init(cpu);
> > +}
> > +}
> > +
> > +if (vms->gic_version == VIRT_GIC_VERSION_2) {
> > +irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
> > + GIC_FDT_IRQ_PPI_CPU_WIDTH,
> > + (1 << vms->smp_cpus) - 1);
> > +}
> > +
> > +armcpu = ARM_CPU(qemu_get_cpu(0));
> > +qemu_fdt_add_subnode(vms->fdt, "/spe");
> > +if (arm_feature(>env, ARM_FEATURE_V8)) {
> > +const char compat[] = "arm,statistical-profiling-extension-v1";
> > +qemu_fdt_setprop(vms->fdt, "/spe", "compatible",
> > + compat, sizeof(compat));
> > +qemu_fdt_setprop_cells(vms->fdt, "/spe", "interrupts",
> > + GIC_FDT_IRQ_TYPE_PPI, VIRTUAL_SPE_IRQ,
> > irqflags);
> > +}
> > +}
>
> Please look at [1]. I've refactored fdt_add_pmu_nodes(), so it no longer
> makes a good pattern for this function.
>
> [1] https://www.mail-archive.com/qemu-devel@nongnu.org/msg727588.html
>
>
Ok, I will spend some time studying your patches, and rework the related patches
in the next version.
> > +
> > static inline DeviceState *create_acpi_ged(VirtMachineState *vms)
> > {
> > DeviceState *dev;
> > @@ -727,6 +763,10 @@ static void create_gic(VirtMachineState *vms)
> > qdev_get_gpio_in(vms->gic, ppibase
> > + VIRTUAL_PMU_IRQ));
> >
> > +qdev_connect_gpio_out_named(cpudev, "spe-interrupt", 0,
> > +qdev_get_gpio_in(vms->gic, ppibase
> > + + VIRTUAL_SPE_IRQ));
> > +
> > sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev,
> > ARM_CPU_IRQ));
> > sysbus_connect_irq(gicbusdev, i + smp_cpus,
> > qdev_get_gpio_in(cpudev, ARM_CPU_FIQ));
> > @@ -1915,6 +1955,8 @@ static void machvirt_init(MachineState *machine)
> >
> > fdt_add_pmu_nodes(vms);
> >
> >
Re: [PATCH 4/7] target/arm/kvm: spe: Unify device attr operatioin helper
On Mon, 10 Aug 2020 at 18:29, Andrew Jones wrote:
>
> On Mon, Aug 10, 2020 at 10:48:41AM +0800, Haibo Xu wrote:
> > On Fri, 7 Aug 2020 at 16:19, Philippe Mathieu-Daudé wrote:
> > >
> > > On 8/7/20 10:10 AM, Haibo Xu wrote:
> > > > Rename kvm_arm_pmu_set_attr() to kvm_arm_dev_set_attr(),
> > >
> > > Maybe rename kvm_arm_device_set_attr() to match the structure
> > > name?
> > >
> >
> > Thanks for the review! I will update it in the next version.
>
> I've already renamed it to kvm_arm_set_device_attr() in [1]. Also, it's
> not enough to just rename the function. The error messages the function
> may generate have "PMU" embedded in them.
>
> [1] https://www.mail-archive.com/qemu-devel@nongnu.org/msg727590.html
>
> Thanks,
> drew
>
Thanks for your review, Andrew!
Will rebase on your patches in the next version.
> >
> > > > So both the vPMU and vSPE device can share the same API.
> > > >
> > > > Signed-off-by: Haibo Xu
> > >
> > > Regardless, with the typo "operation" in patch subject fixed:
> > > Reviewed-by: Philippe Mathieu-Daudé
> > >
> > > > ---
> > > > target/arm/kvm64.c | 6 +++---
> > > > 1 file changed, 3 insertions(+), 3 deletions(-)
> > > >
> > > > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> > > > index 1169237905..75a417d65c 100644
> > > > --- a/target/arm/kvm64.c
> > > > +++ b/target/arm/kvm64.c
> > > > @@ -398,7 +398,7 @@ static CPUWatchpoint *find_hw_watchpoint(CPUState
> > > > *cpu, target_ulong addr)
> > > > return NULL;
> > > > }
> > > >
> > > > -static bool kvm_arm_pmu_set_attr(CPUState *cs, struct kvm_device_attr
> > > > *attr)
> > > > +static bool kvm_arm_dev_set_attr(CPUState *cs, struct kvm_device_attr
> > > > *attr)
> > > > {
> > > > int err;
> > > >
> > > > @@ -427,7 +427,7 @@ void kvm_arm_pmu_init(CPUState *cs)
> > > > if (!ARM_CPU(cs)->has_pmu) {
> > > > return;
> > > > }
> > > > -if (!kvm_arm_pmu_set_attr(cs, )) {
> > > > +if (!kvm_arm_dev_set_attr(cs, )) {
> > > > error_report("failed to init PMU");
> > > > abort();
> > > > }
> > > > @@ -444,7 +444,7 @@ void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
> > > > if (!ARM_CPU(cs)->has_pmu) {
> > > > return;
> > > > }
> > > > -if (!kvm_arm_pmu_set_attr(cs, )) {
> > > > +if (!kvm_arm_dev_set_attr(cs, )) {
> > > > error_report("failed to set irq for PMU");
> > > > abort();
> > > > }
> > > >
> > >
> >
>
Re: [PATCH 3/7] target/arm/cpu: spe: Add an option to turn on/off vSPE support
On Fri, 7 Aug 2020 at 16:28, Philippe Mathieu-Daudé wrote:
>
> Hi Haibo,
>
> On 8/7/20 10:10 AM, Haibo Xu wrote:
> > Adds a spe=[on/off] option to enable/disable vSPE support in
> > guest vCPU. Note this option is only available for "-cpu host"
> > with KVM mode, and default value is on.
> >
> > Signed-off-by: Haibo Xu
> > ---
> > target/arm/cpu.c | 28
> > target/arm/cpu.h | 3 +++
> > 2 files changed, 31 insertions(+)
> >
> > diff --git a/target/arm/cpu.c b/target/arm/cpu.c
> > index 111579554f..40768b4d19 100644
> > --- a/target/arm/cpu.c
> > +++ b/target/arm/cpu.c
> > @@ -1122,6 +1122,29 @@ static void arm_set_pmu(Object *obj, bool value,
> > Error **errp)
> > cpu->has_pmu = value;
> > }
> >
> > +static bool arm_get_spe(Object *obj, Error **errp)
> > +{
> > +ARMCPU *cpu = ARM_CPU(obj);
> > +
> > +return cpu->has_spe;
> > +}
> > +
> > +static void arm_set_spe(Object *obj, bool value, Error **errp)
> > +{
> > +ARMCPU *cpu = ARM_CPU(obj);
> > +
> > +if (value) {
> > +if (kvm_enabled() && !kvm_arm_spe_supported()) {
> > +error_setg(errp, "'spe' feature not supported by KVM on this
> > host");
> > +return;
> > +}
> > +set_feature(>env, ARM_FEATURE_SPE);
> > +} else {
> > +unset_feature(>env, ARM_FEATURE_SPE);
> > +}
> > +cpu->has_spe = value;
> > +}
> > +
> > unsigned int gt_cntfrq_period_ns(ARMCPU *cpu)
> > {
> > /*
> > @@ -1195,6 +1218,11 @@ void arm_cpu_post_init(Object *obj)
> > object_property_add_bool(obj, "pmu", arm_get_pmu, arm_set_pmu);
> > }
> >
> > +if (arm_feature(>env, ARM_FEATURE_SPE)) {
> > +cpu->has_spe = true;
>
> Being a profiling feature, are you sure you want it to be ON by default?
>
> I'd expect the opposite, either being turned on via command line at
> startup or by a management layer at runtime, when someone is ready
> to record the perf events and analyze them.
>
I'm not sure whether it's proper to follow the 'pmu' setting here
which has it on by default.
To be honest, I also prefer to turn it off by default(Please refer to
the comments in the cover letter).
> > +object_property_add_bool(obj, "spe", arm_get_spe, arm_set_spe);
> > +}
> > +
> > /*
> > * Allow user to turn off VFP and Neon support, but only for TCG --
> > * KVM does not currently allow us to lie to the guest about its
> > diff --git a/target/arm/cpu.h b/target/arm/cpu.h
> > index 9e8ed423ea..fe0ac14386 100644
> > --- a/target/arm/cpu.h
> > +++ b/target/arm/cpu.h
> > @@ -822,6 +822,8 @@ struct ARMCPU {
> > bool has_el3;
> > /* CPU has PMU (Performance Monitor Unit) */
> > bool has_pmu;
> > +/* CPU has SPE (Statistical Profiling Extension) */
> > +bool has_spe;
> > /* CPU has VFP */
> > bool has_vfp;
> > /* CPU has Neon */
> > @@ -1959,6 +1961,7 @@ enum arm_features {
> > ARM_FEATURE_VBAR, /* has cp15 VBAR */
> > ARM_FEATURE_M_SECURITY, /* M profile Security Extension */
> > ARM_FEATURE_M_MAIN, /* M profile Main Extension */
> > +ARM_FEATURE_SPE, /* has SPE support */
> > };
> >
> > static inline int arm_feature(CPUARMState *env, int feature)
> >
>
Re: [PATCH 4/7] target/arm/kvm: spe: Unify device attr operatioin helper
On Fri, 7 Aug 2020 at 16:19, Philippe Mathieu-Daudé wrote:
>
> On 8/7/20 10:10 AM, Haibo Xu wrote:
> > Rename kvm_arm_pmu_set_attr() to kvm_arm_dev_set_attr(),
>
> Maybe rename kvm_arm_device_set_attr() to match the structure
> name?
>
Thanks for the review! I will update it in the next version.
> > So both the vPMU and vSPE device can share the same API.
> >
> > Signed-off-by: Haibo Xu
>
> Regardless, with the typo "operation" in patch subject fixed:
> Reviewed-by: Philippe Mathieu-Daudé
>
> > ---
> > target/arm/kvm64.c | 6 +++---
> > 1 file changed, 3 insertions(+), 3 deletions(-)
> >
> > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
> > index 1169237905..75a417d65c 100644
> > --- a/target/arm/kvm64.c
> > +++ b/target/arm/kvm64.c
> > @@ -398,7 +398,7 @@ static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu,
> > target_ulong addr)
> > return NULL;
> > }
> >
> > -static bool kvm_arm_pmu_set_attr(CPUState *cs, struct kvm_device_attr
> > *attr)
> > +static bool kvm_arm_dev_set_attr(CPUState *cs, struct kvm_device_attr
> > *attr)
> > {
> > int err;
> >
> > @@ -427,7 +427,7 @@ void kvm_arm_pmu_init(CPUState *cs)
> > if (!ARM_CPU(cs)->has_pmu) {
> > return;
> > }
> > -if (!kvm_arm_pmu_set_attr(cs, )) {
> > +if (!kvm_arm_dev_set_attr(cs, )) {
> > error_report("failed to init PMU");
> > abort();
> > }
> > @@ -444,7 +444,7 @@ void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
> > if (!ARM_CPU(cs)->has_pmu) {
> > return;
> > }
> > -if (!kvm_arm_pmu_set_attr(cs, )) {
> > +if (!kvm_arm_dev_set_attr(cs, )) {
> > error_report("failed to set irq for PMU");
> > abort();
> > }
> >
>
[PATCH 7/7] target/arm/cpu: spe: Enable spe to work with host cpu
Turn on the spe cpu property by default when working with host
cpu type in KVM mode, i.e. we can now do '-cpu host' to add the
vSPE, and '-cpu host,spe=off' to remove it.
Signed-off-by: Haibo Xu
---
target/arm/cpu.c | 4
target/arm/kvm64.c | 9 +
2 files changed, 13 insertions(+)
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 67ab0089fd..42fa99953c 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -1719,6 +1719,10 @@ static void arm_cpu_realizefn(DeviceState *dev, Error
**errp)
cpu->pmceid1 = 0;
}
+if (!cpu->has_spe || !kvm_enabled()) {
+unset_feature(env, ARM_FEATURE_SPE);
+}
+
if (!arm_feature(env, ARM_FEATURE_EL2)) {
/* Disable the hypervisor feature bits in the processor feature
* registers if we don't have EL2. These are id_pfr1[15:12] and
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index be045ccc5f..4ea58afc1d 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -679,6 +679,7 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
features |= 1ULL << ARM_FEATURE_AARCH64;
features |= 1ULL << ARM_FEATURE_PMU;
features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
+features |= 1ULL << ARM_FEATURE_SPE;
ahcf->features = features;
@@ -826,6 +827,14 @@ int kvm_arch_init_vcpu(CPUState *cs)
} else {
env->features &= ~(1ULL << ARM_FEATURE_PMU);
}
+if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_SPE_V1)) {
+cpu->has_spe = false;
+}
+if (cpu->has_spe) {
+cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SPE_V1;
+} else {
+env->features &= ~(1ULL << ARM_FEATURE_SPE);
+}
if (cpu_isar_feature(aa64_sve, cpu)) {
assert(kvm_arm_sve_supported());
cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
--
2.17.1
[PATCH 2/7] target/arm/kvm: spe: Add helper to detect SPE when using KVM
Signed-off-by: Haibo Xu
---
target/arm/kvm.c | 5 +
target/arm/kvm_arm.h | 13 +
2 files changed, 18 insertions(+)
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 8bb7318378..58f991e890 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -214,6 +214,11 @@ bool kvm_arm_pmu_supported(void)
return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3);
}
+bool kvm_arm_spe_supported(void)
+{
+return kvm_check_extension(kvm_state, KVM_CAP_ARM_SPE_V1);
+}
+
int kvm_arm_get_max_vm_ipa_size(MachineState *ms)
{
KVMState *s = KVM_STATE(ms->accelerator);
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index adb38514bf..f79655674e 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -283,6 +283,14 @@ bool kvm_arm_aarch32_supported(void);
*/
bool kvm_arm_pmu_supported(void);
+/**
+ * kvm_arm_spe_supported:
+ *
+ * Returns: true if the KVM VCPU can enable its SPE
+ * and false otherwise.
+ */
+bool kvm_arm_spe_supported(void);
+
/**
* kvm_arm_sve_supported:
*
@@ -366,6 +374,11 @@ static inline bool kvm_arm_pmu_supported(void)
return false;
}
+static inline bool kvm_arm_spe_supported(void)
+{
+return false;
+}
+
static inline bool kvm_arm_sve_supported(void)
{
return false;
--
2.17.1
[PATCH 5/7] target/arm/kvm: spe: Add device init and set_irq operations
Signed-off-by: Haibo Xu
---
target/arm/kvm64.c | 33 +
target/arm/kvm_arm.h | 5 +
2 files changed, 38 insertions(+)
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 75a417d65c..be045ccc5f 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -450,6 +450,39 @@ void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
}
}
+void kvm_arm_spe_init(CPUState *cs)
+{
+struct kvm_device_attr attr = {
+.group = KVM_ARM_VCPU_SPE_V1_CTRL,
+.attr = KVM_ARM_VCPU_SPE_V1_INIT,
+};
+
+if (!ARM_CPU(cs)->has_spe) {
+return;
+}
+if (!kvm_arm_dev_set_attr(cs, )) {
+error_report("failed to init SPE");
+abort();
+}
+}
+
+void kvm_arm_spe_set_irq(CPUState *cs, int irq)
+{
+struct kvm_device_attr attr = {
+.group = KVM_ARM_VCPU_SPE_V1_CTRL,
+.addr = (intptr_t),
+.attr = KVM_ARM_VCPU_SPE_V1_IRQ,
+};
+
+if (!ARM_CPU(cs)->has_spe) {
+return;
+}
+if (!kvm_arm_dev_set_attr(cs, )) {
+error_report("failed to set irq for SPE");
+abort();
+}
+}
+
static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id)
{
uint64_t ret;
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index f79655674e..bb155322eb 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -348,6 +348,8 @@ int kvm_arm_vgic_probe(void);
void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
void kvm_arm_pmu_init(CPUState *cs);
+void kvm_arm_spe_set_irq(CPUState *cs, int irq);
+void kvm_arm_spe_init(CPUState *cs);
int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level);
#else
@@ -397,6 +399,9 @@ static inline int kvm_arm_vgic_probe(void)
static inline void kvm_arm_pmu_set_irq(CPUState *cs, int irq) {}
static inline void kvm_arm_pmu_init(CPUState *cs) {}
+static inline void kvm_arm_spe_set_irq(CPUState *cs, int irq) {}
+static inline void kvm_arm_spe_init(CPUState *cs) {}
+
static inline void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) {}
static inline void kvm_arm_get_virtual_time(CPUState *cs) {}
--
2.17.1
