Author: jhb
Date: Fri Sep 30 01:42:29 2016
New Revision: 306472
URL: https://svnweb.freebsd.org/changeset/base/306472
Log:
MFC 304858,305485: Fix various issues with PCI pass through and VT-d.
304858:
Enable I/O MMU when PCI pass through is first used.
Rather than enabling the I/O MMU when the vmm module is loaded,
defer initialization until the first attempt to pass a PCI device
through to a guest. If the I/O MMU fails to initialize or is not
present, than fail the attempt to pass a PCI device through to a
guest.
The hw.vmm.force_iommu tunable has been removed since the I/O MMU is
no longer enabled during boot. However, the I/O MMU support can be
disabled by setting the hw.vmm.iommu.enable tunable to 0 to prevent
use of the I/O MMU on any systems where it is buggy.
305485:
Leave ppt devices in the host domain when they are not attached to a VM.
This allows a pass through device to be reset to a normal device driver
on the host and reused on the host. ppt devices are now always active in
some I/O MMU domain when the I/O MMU is active, either the host domain
or the domain of a VM they are attached to.
Modified:
stable/10/sys/amd64/vmm/io/iommu.c
stable/10/sys/amd64/vmm/io/iommu.h
stable/10/sys/amd64/vmm/io/ppt.c
stable/10/sys/amd64/vmm/vmm.c
Directory Properties:
stable/10/ (props changed)
Modified: stable/10/sys/amd64/vmm/io/iommu.c
==============================================================================
--- stable/10/sys/amd64/vmm/io/iommu.c Fri Sep 30 01:39:18 2016
(r306471)
+++ stable/10/sys/amd64/vmm/io/iommu.c Fri Sep 30 01:42:29 2016
(r306472)
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
+#include <machine/cpu.h>
#include <machine/md_var.h>
#include "vmm_util.h"
@@ -51,6 +52,10 @@ static int iommu_avail;
SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail,
0, "bhyve iommu initialized?");
+static int iommu_enable = 1;
+SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
+ "Enable use of I/O MMU (required for PCI passthrough).");
+
static struct iommu_ops *ops;
static void *host_domain;
@@ -148,14 +153,16 @@ IOMMU_DISABLE(void)
(*ops->disable)();
}
-void
+static void
iommu_init(void)
{
int error, bus, slot, func;
vm_paddr_t maxaddr;
- const char *name;
device_t dev;
+ if (!iommu_enable)
+ return;
+
if (vmm_is_intel())
ops = &iommu_ops_intel;
else if (vmm_is_amd())
@@ -174,8 +181,13 @@ iommu_init(void)
*/
maxaddr = vmm_mem_maxaddr();
host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
- if (host_domain == NULL)
- panic("iommu_init: unable to create a host domain");
+ if (host_domain == NULL) {
+ printf("iommu_init: unable to create a host domain");
+ IOMMU_CLEANUP();
+ ops = NULL;
+ iommu_avail = 0;
+ return;
+ }
/*
* Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
@@ -190,12 +202,7 @@ iommu_init(void)
if (dev == NULL)
continue;
- /* skip passthrough devices */
- name = device_get_name(dev);
- if (name != NULL && strcmp(name, "ppt") == 0)
- continue;
-
- /* everything else belongs to the host domain */
+ /* Everything belongs to the host domain. */
iommu_add_device(host_domain,
pci_get_rid(dev));
}
@@ -216,7 +223,16 @@ iommu_cleanup(void)
void *
iommu_create_domain(vm_paddr_t maxaddr)
{
+ static volatile int iommu_initted;
+ if (iommu_initted < 2) {
+ if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
+ iommu_init();
+ atomic_store_rel_int(&iommu_initted, 2);
+ } else
+ while (iommu_initted == 1)
+ cpu_spinwait();
+ }
return (IOMMU_CREATE_DOMAIN(maxaddr));
}
Modified: stable/10/sys/amd64/vmm/io/iommu.h
==============================================================================
--- stable/10/sys/amd64/vmm/io/iommu.h Fri Sep 30 01:39:18 2016
(r306471)
+++ stable/10/sys/amd64/vmm/io/iommu.h Fri Sep 30 01:42:29 2016
(r306472)
@@ -61,7 +61,6 @@ struct iommu_ops {
extern struct iommu_ops iommu_ops_intel;
extern struct iommu_ops iommu_ops_amd;
-void iommu_init(void);
void iommu_cleanup(void);
void *iommu_host_domain(void);
void *iommu_create_domain(vm_paddr_t maxaddr);
Modified: stable/10/sys/amd64/vmm/io/ppt.c
==============================================================================
--- stable/10/sys/amd64/vmm/io/ppt.c Fri Sep 30 01:39:18 2016
(r306471)
+++ stable/10/sys/amd64/vmm/io/ppt.c Fri Sep 30 01:42:29 2016
(r306472)
@@ -363,6 +363,7 @@ ppt_assign_device(struct vm *vm, int bus
return (EBUSY);
ppt->vm = vm;
+ iommu_remove_device(iommu_host_domain(), pci_get_rid(ppt->dev));
iommu_add_device(vm_iommu_domain(vm), pci_get_rid(ppt->dev));
return (0);
}
@@ -385,6 +386,7 @@ ppt_unassign_device(struct vm *vm, int b
ppt_teardown_msi(ppt);
ppt_teardown_msix(ppt);
iommu_remove_device(vm_iommu_domain(vm), pci_get_rid(ppt->dev));
+ iommu_add_device(iommu_host_domain(), pci_get_rid(ppt->dev));
ppt->vm = NULL;
return (0);
}
Modified: stable/10/sys/amd64/vmm/vmm.c
==============================================================================
--- stable/10/sys/amd64/vmm/vmm.c Fri Sep 30 01:39:18 2016
(r306471)
+++ stable/10/sys/amd64/vmm/vmm.c Fri Sep 30 01:42:29 2016
(r306472)
@@ -228,11 +228,6 @@ SYSCTL_INT(_hw_vmm, OID_AUTO, trace_gues
&trace_guest_exceptions, 0,
"Trap into hypervisor on all guest exceptions and reflect them back");
-static int vmm_force_iommu = 0;
-TUNABLE_INT("hw.vmm.force_iommu", &vmm_force_iommu);
-SYSCTL_INT(_hw_vmm, OID_AUTO, force_iommu, CTLFLAG_RDTUN, &vmm_force_iommu, 0,
- "Force use of I/O MMU even if no passthrough devices were found.");
-
static void vm_free_memmap(struct vm *vm, int ident);
static bool sysmem_mapping(struct vm *vm, struct mem_map *mm);
static void vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr);
@@ -362,8 +357,6 @@ vmm_handler(module_t mod, int what, void
switch (what) {
case MOD_LOAD:
vmmdev_init();
- if (vmm_force_iommu || ppt_avail_devices() > 0)
- iommu_init();
error = vmm_init();
if (error == 0)
vmm_initialized = 1;
@@ -400,9 +393,6 @@ static moduledata_t vmm_kmod = {
/*
* vmm initialization has the following dependencies:
*
- * - iommu initialization must happen after the pci passthru driver has had
- * a chance to attach to any passthru devices (after SI_SUB_CONFIGURE).
- *
* - VT-x initialization requires smp_rendezvous() and therefore must happen
* after SMP is fully functional (after SI_SUB_SMP).
*/
@@ -897,6 +887,8 @@ vm_assign_pptdev(struct vm *vm, int bus,
("vm_assign_pptdev: iommu must be NULL"));
maxaddr = sysmem_maxaddr(vm);
vm->iommu = iommu_create_domain(maxaddr);
+ if (vm->iommu == NULL)
+ return (ENXIO);
vm_iommu_map(vm);
}
_______________________________________________
svn-src-stable-10@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-stable-10
To unsubscribe, send any mail to "svn-src-stable-10-unsubscr...@freebsd.org"
