In order to add a virtual PCI host controller to the root cell after Jailhouse activation on ARM/ARM64, we need to inject a device tree modification. For this purpose, overlays were invented and are already support by upstream since 3.19.
All we need to do is to fill a fragment template with some variable parameters that can be derived from the system configuration: virtual IRQ base, ECAM base address, and the uncached memory window size, which can be derived from the number of ivshmem devices. We also need to resolve the link to the GIC as the host's device tree may not have the required symbols included. Once the fragment is patched, we can register it, and Linux will detect and handle the new platform device. Note that the upstream pci-host-generic Linux driver still has issues when it comes to unloading (and actually also rollback on errors). Fixing this is WiP. To use this feature with the Tegra K1, the kernel has to be patched to allow multi-domain PCI so that a second PCI host controller can be enabled besides the non-extensible physical controller (see https://groups.google.com/d/msg/jailhouse-dev/5BnCAFAzykI/wHgj2UUbCwAJ). Signed-off-by: Jan Kiszka <[email protected]> --- driver/Makefile | 7 +++ driver/main.c | 2 +- driver/pci.c | 146 ++++++++++++++++++++++++++++++++++++++++++++++- driver/pci.h | 5 +- driver/vpci_template.dts | 26 +++++++++ 5 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 driver/vpci_template.dts diff --git a/driver/Makefile b/driver/Makefile index e8d3ac5..6b7e986 100644 --- a/driver/Makefile +++ b/driver/Makefile @@ -18,5 +18,12 @@ ccflags-y := -I$(src)/../hypervisor/arch/$(SRCARCH)/include \ jailhouse-y := cell.o main.o sysfs.o jailhouse-$(CONFIG_PCI) += pci.o +jailhouse-$(CONFIG_OF) += vpci_template.dtb.o $(obj)/main.o: $(obj)/../hypervisor/include/generated/version.h + +targets += vpci_template.dtb vpci_template.dtb.S + +.SECONDARY: \ + $(obj)/vpci_template.dtb.S \ + $(obj)/vpci_template.dtb diff --git a/driver/main.c b/driver/main.c index 2c6f1c7..206e29f 100644 --- a/driver/main.c +++ b/driver/main.c @@ -341,7 +341,7 @@ static int jailhouse_cmd_enable(struct jailhouse_system __user *arg) release_firmware(hypervisor); jailhouse_cell_register_root(); - jailhouse_pci_virtual_root_devices_add(); + jailhouse_pci_virtual_root_devices_add(&config_header); jailhouse_enabled = true; diff --git a/driver/pci.c b/driver/pci.c index 9a95856..f24adbc 100644 --- a/driver/pci.c +++ b/driver/pci.c @@ -13,7 +13,10 @@ #include <linux/list.h> #include <linux/pci.h> +#include <linux/of.h> +#include <linux/of_fdt.h> #include <linux/vmalloc.h> +#include <linux/version.h> #include "pci.h" @@ -228,8 +231,147 @@ void jailhouse_pci_cell_cleanup(struct cell *cell) vfree(cell->pci_devices); } -void jailhouse_pci_virtual_root_devices_add(void) +#ifdef CONFIG_OF +extern u8 __dtb_vpci_template_begin[], __dtb_vpci_template_end[]; + +static int overlay_id = -1; + +static unsigned int count_ivshmem_devices(struct cell *cell) +{ + const struct jailhouse_pci_device *dev = cell->pci_devices; + unsigned int n, count = 0; + + for (n = cell->num_pci_devices; n > 0; n--, dev++) + if (dev->type == JAILHOUSE_PCI_TYPE_IVSHMEM) + count++; + return count; +} + +static const struct of_device_id gic_of_match[] = { + { .compatible = "arm,cortex-a15-gic", }, + { .compatible = "arm,cortex-a7-gic", }, + { .compatible = "arm,gic-400", }, + {}, +}; + +static bool create_vpci_of_overlay(struct jailhouse_system *config) +{ + const size_t size = __dtb_vpci_template_end - __dtb_vpci_template_begin; + struct device_node *overlay = NULL; + struct device_node *vpci = NULL; + struct device_node *gic = NULL; + void *overlay_data = NULL; + bool success = false; + phandle gic_handle; + u32 *prop_val; + u64 base_addr; + int len; + + overlay_data = kmemdup(__dtb_vpci_template_begin, size, GFP_KERNEL); + if (!overlay_data) + goto out; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,7,0) + of_fdt_unflatten_tree(overlay_data, NULL, &overlay); +#else + of_fdt_unflatten_tree(overlay_data, &overlay); +#endif + if (!overlay) + goto out; + + of_node_set_flag(overlay, OF_DETACHED); + + gic = of_find_matching_node(NULL, gic_of_match); + if (!gic) + goto out; + + gic_handle = cpu_to_be32(gic->phandle); + + vpci = of_find_node_by_name(overlay, "vpci"); + if (!vpci) + goto out; + + prop_val = (u32 *)of_get_property(vpci, "interrupt-map", &len); + if (!prop_val || len != 4 * 8 * sizeof(u32)) + goto out; + + /* + * Inject the GIC handles and the slot's SPI number in the interrupt + * map. We can patch this tree as it's still local in our overlay_data. + */ + prop_val[4] = gic_handle; + prop_val[6] = cpu_to_be32(config->root_cell.vpci_irq_base); + prop_val[12] = gic_handle; + prop_val[14] = cpu_to_be32(config->root_cell.vpci_irq_base + 1); + prop_val[20] = gic_handle; + prop_val[22] = cpu_to_be32(config->root_cell.vpci_irq_base + 2); + prop_val[28] = gic_handle; + prop_val[30] = cpu_to_be32(config->root_cell.vpci_irq_base + 3); + + prop_val = (u32 *)of_get_property(vpci, "reg", &len); + if (!prop_val || len != 4 * sizeof(u32)) + goto out; + + /* Set the MMCONFIG base address of the host controller. */ + base_addr = config->platform_info.pci_mmconfig_base; + prop_val[0] = cpu_to_be32(base_addr >> 32); + prop_val[1] = cpu_to_be32(base_addr); + + prop_val = (u32 *)of_get_property(vpci, "ranges", &len); + if (!prop_val || len != 7 * sizeof(u32)) + goto out; + + /* + * Locate the resource window right after MMCONFIG, which is only + * covering one bus. Reserve 2 pages per virtual shared memory device. + */ + base_addr += 0x100000; + prop_val[1] = cpu_to_be32(base_addr >> 32); + prop_val[2] = cpu_to_be32(base_addr); + prop_val[3] = cpu_to_be32(base_addr >> 32); + prop_val[4] = cpu_to_be32(base_addr); + prop_val[6] = cpu_to_be32(count_ivshmem_devices(root_cell) * 0x2000); + + overlay_id = of_overlay_create(overlay); + if (overlay_id < 0) + goto out; + + of_node_put(overlay); + + success = true; + +out: + of_node_put(vpci); + of_node_put(gic); + kfree(overlay_data); + + return success; +} + +static void destroy_vpci_of_overlay(void) +{ + if (overlay_id >= 0) + of_overlay_destroy(overlay_id); +} +#else /* !CONFIG_OF */ +static bool create_vpci_of_overlay(struct jailhouse_system *config) { + return false; +} + +static void destroy_vpci_of_overlay(void) +{ +} +#endif + +void jailhouse_pci_virtual_root_devices_add(struct jailhouse_system *config) +{ + if (config->platform_info.pci_is_virtual && + !create_vpci_of_overlay(config)) { + pr_warn("jailhouse: failed to add virtual host controller\n"); + return; + } + jailhouse_pci_do_all_devices(root_cell, JAILHOUSE_PCI_TYPE_IVSHMEM, JAILHOUSE_PCI_ACTION_ADD); } @@ -238,4 +380,6 @@ void jailhouse_pci_virtual_root_devices_remove(void) { jailhouse_pci_do_all_devices(root_cell, JAILHOUSE_PCI_TYPE_IVSHMEM, JAILHOUSE_PCI_ACTION_DEL); + + destroy_vpci_of_overlay(); } diff --git a/driver/pci.h b/driver/pci.h index 6b358c5..598c6c8 100644 --- a/driver/pci.h +++ b/driver/pci.h @@ -25,7 +25,7 @@ void jailhouse_pci_do_all_devices(struct cell *cell, unsigned int type, int jailhouse_pci_cell_setup(struct cell *cell, const struct jailhouse_cell_desc *cell_desc); void jailhouse_pci_cell_cleanup(struct cell *cell); -void jailhouse_pci_virtual_root_devices_add(void); +void jailhouse_pci_virtual_root_devices_add(struct jailhouse_system *config); void jailhouse_pci_virtual_root_devices_remove(void); int jailhouse_pci_register(void); void jailhouse_pci_unregister(void); @@ -49,7 +49,8 @@ static inline void jailhouse_pci_cell_cleanup(struct cell *cell) { } -static inline void jailhouse_pci_virtual_root_devices_add(void) +static inline void +jailhouse_pci_virtual_root_devices_add(struct jailhouse_system *config) { } diff --git a/driver/vpci_template.dts b/driver/vpci_template.dts new file mode 100644 index 0000000..5a0dccd --- /dev/null +++ b/driver/vpci_template.dts @@ -0,0 +1,26 @@ +/dts-v1/; +/ { + fragment { + target-path = "/"; + __overlay__ { + #address-cells = <2>; + #size-cells = <2>; + + vpci@0 { + compatible = "pci-host-ecam-generic"; + device_type = "pci"; + bus-range = <0 0>; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 0xbad 0 0xbad 0>, + <0 0 0 2 0xbad 0 0xbad 0>, + <0 0 0 3 0xbad 0 0xbad 0>, + <0 0 0 4 0xbad 0 0xbad 0>; + reg = <0xbad 0xbad 0x0 0x100000>; + ranges = <0x02000000 0xbad 0xbad 0xbad 0xbad 0x0 0xbad>; + }; + }; + }; +}; -- 2.1.4 -- You received this message because you are subscribed to the Google Groups "Jailhouse" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/d/optout.
