From: Jan Kiszka <[email protected]> This is easy for the virtual ivshmem device but more complex and, thus, not complete for physical ones: Reset the state of assigned PCI devices when the owner cell is started, which includes restarts (load/start cycles), or when ownership changes (physical devices only).
For both device types, we bring all MSI/MSI-X vectors into reset state. Physical devices get their INTx line masked, just like before on device removal. Shared memory devices additional reinitialize their config space and MMIO registers. Signed-off-by: Jan Kiszka <[email protected]> --- hypervisor/control.c | 1 + hypervisor/include/jailhouse/ivshmem.h | 2 ++ hypervisor/include/jailhouse/pci.h | 3 ++ hypervisor/ivshmem.c | 54 ++++++++++++++++++++++------------ hypervisor/pci.c | 45 ++++++++++++++++++++++++++-- 5 files changed, 85 insertions(+), 20 deletions(-) diff --git a/hypervisor/control.c b/hypervisor/control.c index d6a5a7d..b1beed9 100644 --- a/hypervisor/control.c +++ b/hypervisor/control.c @@ -558,6 +558,7 @@ static int cell_start(struct per_cpu *cpu_data, unsigned long id) cell->comm_page.comm_region.cell_state = JAILHOUSE_CELL_RUNNING; cell->comm_page.comm_region.msg_to_cell = JAILHOUSE_MSG_NONE; + pci_cell_reset(cell); arch_cell_reset(cell); for_each_cpu(cpu, cell->cpu_set) { diff --git a/hypervisor/include/jailhouse/ivshmem.h b/hypervisor/include/jailhouse/ivshmem.h index 82a4837..8872876 100644 --- a/hypervisor/include/jailhouse/ivshmem.h +++ b/hypervisor/include/jailhouse/ivshmem.h @@ -35,6 +35,7 @@ struct ivshmem_endpoint { u64 bar0_address; u64 bar4_address; struct pci_device *device; + const struct jailhouse_memory *shmem; struct ivshmem_endpoint *remote; spinlock_t remote_lock; struct arch_pci_ivshmem arch; @@ -42,6 +43,7 @@ struct ivshmem_endpoint { }; int ivshmem_init(struct cell *cell, struct pci_device *device); +void ivshmem_reset(struct pci_device *device); void ivshmem_exit(struct pci_device *device); int ivshmem_update_msix(struct pci_device *device); enum pci_access ivshmem_pci_cfg_write(struct pci_device *device, diff --git a/hypervisor/include/jailhouse/pci.h b/hypervisor/include/jailhouse/pci.h index 7859372..c0c10e9 100644 --- a/hypervisor/include/jailhouse/pci.h +++ b/hypervisor/include/jailhouse/pci.h @@ -159,7 +159,10 @@ enum pci_access pci_cfg_read_moderate(struct pci_device *device, u16 address, enum pci_access pci_cfg_write_moderate(struct pci_device *device, u16 address, unsigned int size, u32 value); +void pci_reset_device(struct pci_device *device); + int pci_cell_init(struct cell *cell); +void pci_cell_reset(struct cell *cell); void pci_cell_exit(struct cell *cell); void pci_config_commit(struct cell *cell_added_removed); diff --git a/hypervisor/ivshmem.c b/hypervisor/ivshmem.c index a9259cf..cfe7d0a 100644 --- a/hypervisor/ivshmem.c +++ b/hypervisor/ivshmem.c @@ -366,38 +366,56 @@ int ivshmem_init(struct cell *cell, struct pci_device *device) ive = &iv->eps[id]; remote = &iv->eps[id ^ 1]; + ive->device = device; + ive->shmem = mem; + ive->ivpos = id; + device->ivshmem_endpoint = ive; + if (remote->device) { + ive->remote = remote; + remote->remote = ive; + } + + device->cell = cell; + pci_reset_device(device); + + return 0; +} + +void ivshmem_reset(struct pci_device *device) +{ + struct ivshmem_endpoint *ive = device->ivshmem_endpoint; + + if (ive->cspace[PCI_CFG_COMMAND/4] & PCI_CMD_MEM) { + mmio_region_unregister(device->cell, ive->bar0_address); + mmio_region_unregister(device->cell, ive->bar4_address); + } + + memset(device->bar, 0, sizeof(device->bar)); + device->msix_registers.raw = 0; + device->bar[0] = PCI_BAR_64BIT; memcpy(ive->cspace, &default_cspace, sizeof(default_cspace)); - ive->cspace[0x08/4] |= dev_info->shmem_protocol << 8; + ive->cspace[0x08/4] |= device->info->shmem_protocol << 8; - if (dev_info->num_msix_vectors == 0) { + if (device->info->num_msix_vectors == 0) { /* let the PIN rotate based on the device number */ ive->cspace[PCI_CFG_INT/4] = - (((dev_info->bdf >> 3) & 0x3) + 1) << 8; + (((device->info->bdf >> 3) & 0x3) + 1) << 8; /* disable MSI-X capability */ ive->cspace[PCI_CFG_CAPS/4] = 0; } else { device->bar[4] = PCI_BAR_64BIT; } - ive->cspace[IVSHMEM_CFG_SHMEM_PTR/4] = (u32)mem->virt_start; - ive->cspace[IVSHMEM_CFG_SHMEM_PTR/4 + 1] = (u32)(mem->virt_start >> 32); - ive->cspace[IVSHMEM_CFG_SHMEM_SZ/4] = (u32)mem->size; - ive->cspace[IVSHMEM_CFG_SHMEM_SZ/4 + 1] = (u32)(mem->size >> 32); + ive->cspace[IVSHMEM_CFG_SHMEM_PTR/4] = (u32)ive->shmem->virt_start; + ive->cspace[IVSHMEM_CFG_SHMEM_PTR/4 + 1] = + (u32)(ive->shmem->virt_start >> 32); + ive->cspace[IVSHMEM_CFG_SHMEM_SZ/4] = (u32)ive->shmem->size; + ive->cspace[IVSHMEM_CFG_SHMEM_SZ/4 + 1] = (u32)(ive->shmem->size >> 32); - ive->device = device; - ive->ivpos = id; - device->ivshmem_endpoint = ive; - if (remote->device) { - ive->remote = remote; - remote->remote = ive; - } - - device->cell = cell; - - return 0; + ive->state = 0; } /** diff --git a/hypervisor/pci.c b/hypervisor/pci.c index 3df3c83..913fde5 100644 --- a/hypervisor/pci.c +++ b/hypervisor/pci.c @@ -16,6 +16,7 @@ #include <jailhouse/mmio.h> #include <jailhouse/pci.h> #include <jailhouse/printk.h> +#include <jailhouse/string.h> #include <jailhouse/utils.h> #define MSIX_VECTOR_CTRL_DWORD 3 @@ -552,6 +553,36 @@ void pci_prepare_handover(void) } } +void pci_reset_device(struct pci_device *device) +{ + const struct jailhouse_pci_capability *cap; + unsigned int n; + + memset(&device->msi_registers, 0, sizeof(device->msi_registers)); + for (n = 0; n < device->info->num_msix_vectors; n++) { + device->msix_vectors[n].address = 0; + device->msix_vectors[n].data = 0; + device->msix_vectors[n].masked = 1; + } + + if (device->info->type == JAILHOUSE_PCI_TYPE_IVSHMEM) { + ivshmem_reset(device); + return; + } + + /* + * Silence INTx of the physical device by setting the mask bit. + * This is a deviation from the specified reset state. + */ + pci_write_config(device->info->bdf, PCI_CFG_COMMAND, + PCI_CMD_INTX_OFF, 2); + + for_each_pci_cap(cap, device, n) + if (cap->id == PCI_CAP_MSI || cap->id == PCI_CAP_MSIX) + /* Disable MSI/MSI-X by clearing the control word. */ + pci_write_config(device->info->bdf, cap->start+2, 0, 2); +} + static int pci_add_physical_device(struct cell *cell, struct pci_device *device) { unsigned int n, pages, size = device->info->msix_region_size; @@ -591,6 +622,8 @@ static int pci_add_physical_device(struct cell *cell, struct pci_device *device) } device->cell = cell; + if (cell != &root_cell) + pci_reset_device(device); return 0; @@ -609,9 +642,8 @@ static void pci_remove_physical_device(struct pci_device *device) printk("Removing PCI device %02x:%02x.%x from cell \"%s\"\n", PCI_BDF_PARAMS(device->info->bdf), cell->config->name); + pci_reset_device(device); arch_pci_remove_physical_device(device); - pci_write_config(device->info->bdf, PCI_CFG_COMMAND, - PCI_CMD_INTX_OFF, 2); device->cell = NULL; @@ -703,6 +735,15 @@ error: return err; } +void pci_cell_reset(struct cell *cell) +{ + struct pci_device *device; + + for_each_configured_pci_device(device, cell) + if (device->cell) + pci_reset_device(device); +} + static void pci_return_device_to_root_cell(struct pci_device *device) { struct pci_device *root_device; -- 2.1.4 -- You received this message because you are subscribed to the Google Groups "Jailhouse" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/d/optout.
