Re: svn commit: r349184 - head/sys/amd64/vmm/intel
> Author: scottl
> Date: Wed Jun 19 06:41:07 2019
> New Revision: 349184
> URL: https://svnweb.freebsd.org/changeset/base/349184
>
> Log:
> Implement VT-d capability detection on chipsets that have multiple
> translation units with differing capabilities
>
> From the author via Bugzilla:
> ---
If you had read the full bug report you would also know:
https://reviews.freebsd.org/D19001
existed and that some code cleanup had occurred since
this bug was created.
The review was pending approval by bhyve maintainer(s).
> When an attempt is made to passthrough a PCI device to a bhyve VM
> (causing initialisation of IOMMU) on certain Intel chipsets using
> VT-d the PCI bus stops working entirely. This issue occurs on the
> E3-1275 v5 processor on C236 chipset and has also been encountered
> by others on the forums with different hardware in the Skylake
> series.
>
> The chipset has two VT-d translation units. The issue is caused by
> an attempt to use the VT-d device-IOTLB capability that is
> supported by only the first unit for devices attached to the
> second unit which lacks that capability. Only the capabilities of
> the first unit are checked and are assumed to be the same for all
> units.
>
> Attached is a patch to rectify this issue by determining which
> unit is responsible for the device being added to a domain and
> then checking that unit's device-IOTLB capability. In addition to
> this a few fixes have been made to other instances where the first
> unit's capabilities are assumed for all units for domains they
> share. In these cases a mutual set of capabilities is determined.
> The patch should hopefully fix any bugs for current/future
> hardware with multiple translation units supporting different
> capabilities.
>
> A description is on the forums at
> https://forums.freebsd.org/threads/pci-passthrough-bhyve-usb-xhci.65235
> The thread includes observations by other users of the bug
> occurring, and description as well as confirmation of the fix.
> I'd also like to thank Ordoban for their help.
>
> ---
> Personally tested on a Skylake laptop, Skylake Xeon server, and
> a Xeon-D-1541, passing through XHCI and NVMe functions. Passthru
> is hit-or-miss to the point of being unusable without this
> patch.
>
> PR: 229852
> Submitted by: cal...@aitchison.org
> MFC after: 1 week
>
> Modified:
> head/sys/amd64/vmm/intel/vtd.c
>
> Modified: head/sys/amd64/vmm/intel/vtd.c
> ==
> --- head/sys/amd64/vmm/intel/vtd.cWed Jun 19 03:33:00 2019
> (r349183)
> +++ head/sys/amd64/vmm/intel/vtd.cWed Jun 19 06:41:07 2019
> (r349184)
> @@ -51,6 +51,8 @@ __FBSDID("$FreeBSD$");
> * Architecture Spec, September 2008.
> */
>
> +#define VTD_DRHD_INCLUDE_PCI_ALL(Flags) (((Flags) >> 0) & 0x1)
> +
> /* Section 10.4 "Register Descriptions" */
> struct vtdmap {
> volatile uint32_t version;
> @@ -116,10 +118,11 @@ struct domain {
> static SLIST_HEAD(, domain) domhead;
>
> #define DRHD_MAX_UNITS 8
> -static int drhd_num;
> -static struct vtdmap *vtdmaps[DRHD_MAX_UNITS];
> -static int max_domains;
> -typedef int (*drhd_ident_func_t)(void);
> +static ACPI_DMAR_HARDWARE_UNIT *drhds[DRHD_MAX_UNITS];
> +static int drhd_num;
> +static struct vtdmap *vtdmaps[DRHD_MAX_UNITS];
> +static int max_domains;
> +typedef int (*drhd_ident_func_t)(void);
>
> static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
> static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)]
> __aligned(4096);
> @@ -175,6 +178,69 @@ domain_id(void)
> return (id);
> }
>
> +static struct vtdmap *
> +vtd_device_scope(uint16_t rid)
> +{
> + int i, remaining, pathremaining;
> + char *end, *pathend;
> + struct vtdmap *vtdmap;
> + ACPI_DMAR_HARDWARE_UNIT *drhd;
> + ACPI_DMAR_DEVICE_SCOPE *device_scope;
> + ACPI_DMAR_PCI_PATH *path;
> +
> + for (i = 0; i < drhd_num; i++) {
> + drhd = drhds[i];
> +
> + if (VTD_DRHD_INCLUDE_PCI_ALL(drhd->Flags)) {
> + /*
> + * From Intel VT-d arch spec, version 3.0:
> + * If a DRHD structure with INCLUDE_PCI_ALL flag Set is
> reported
> + * for a Segment, it must be enumerated by BIOS after
> all other
> + * DRHD structures for the same Segment.
> + */
> + vtdmap = vtdmaps[i];
> + return(vtdmap);
> + }
> +
> + end = (char *)drhd + drhd->Header.Length;
> + remaining = drhd->Header.Length -
> sizeof(ACPI_DMAR_HARDWARE_UNIT);
> + while (remaining > sizeof(ACPI_DMAR_DEVICE_SCOPE)) {
> + device_scope =
svn commit: r349184 - head/sys/amd64/vmm/intel
Author: scottl
Date: Wed Jun 19 06:41:07 2019
New Revision: 349184
URL: https://svnweb.freebsd.org/changeset/base/349184
Log:
Implement VT-d capability detection on chipsets that have multiple
translation units with differing capabilities
From the author via Bugzilla:
---
When an attempt is made to passthrough a PCI device to a bhyve VM
(causing initialisation of IOMMU) on certain Intel chipsets using
VT-d the PCI bus stops working entirely. This issue occurs on the
E3-1275 v5 processor on C236 chipset and has also been encountered
by others on the forums with different hardware in the Skylake
series.
The chipset has two VT-d translation units. The issue is caused by
an attempt to use the VT-d device-IOTLB capability that is
supported by only the first unit for devices attached to the
second unit which lacks that capability. Only the capabilities of
the first unit are checked and are assumed to be the same for all
units.
Attached is a patch to rectify this issue by determining which
unit is responsible for the device being added to a domain and
then checking that unit's device-IOTLB capability. In addition to
this a few fixes have been made to other instances where the first
unit's capabilities are assumed for all units for domains they
share. In these cases a mutual set of capabilities is determined.
The patch should hopefully fix any bugs for current/future
hardware with multiple translation units supporting different
capabilities.
A description is on the forums at
https://forums.freebsd.org/threads/pci-passthrough-bhyve-usb-xhci.65235
The thread includes observations by other users of the bug
occurring, and description as well as confirmation of the fix.
I'd also like to thank Ordoban for their help.
---
Personally tested on a Skylake laptop, Skylake Xeon server, and
a Xeon-D-1541, passing through XHCI and NVMe functions. Passthru
is hit-or-miss to the point of being unusable without this
patch.
PR: 229852
Submitted by: cal...@aitchison.org
MFC after: 1 week
Modified:
head/sys/amd64/vmm/intel/vtd.c
Modified: head/sys/amd64/vmm/intel/vtd.c
==
--- head/sys/amd64/vmm/intel/vtd.c Wed Jun 19 03:33:00 2019
(r349183)
+++ head/sys/amd64/vmm/intel/vtd.c Wed Jun 19 06:41:07 2019
(r349184)
@@ -51,6 +51,8 @@ __FBSDID("$FreeBSD$");
* Architecture Spec, September 2008.
*/
+#define VTD_DRHD_INCLUDE_PCI_ALL(Flags) (((Flags) >> 0) & 0x1)
+
/* Section 10.4 "Register Descriptions" */
struct vtdmap {
volatile uint32_t version;
@@ -116,10 +118,11 @@ struct domain {
static SLIST_HEAD(, domain) domhead;
#defineDRHD_MAX_UNITS 8
-static int drhd_num;
-static struct vtdmap *vtdmaps[DRHD_MAX_UNITS];
-static int max_domains;
-typedef int(*drhd_ident_func_t)(void);
+static ACPI_DMAR_HARDWARE_UNIT *drhds[DRHD_MAX_UNITS];
+static int drhd_num;
+static struct vtdmap *vtdmaps[DRHD_MAX_UNITS];
+static int max_domains;
+typedef int(*drhd_ident_func_t)(void);
static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
@@ -175,6 +178,69 @@ domain_id(void)
return (id);
}
+static struct vtdmap *
+vtd_device_scope(uint16_t rid)
+{
+ int i, remaining, pathremaining;
+ char *end, *pathend;
+ struct vtdmap *vtdmap;
+ ACPI_DMAR_HARDWARE_UNIT *drhd;
+ ACPI_DMAR_DEVICE_SCOPE *device_scope;
+ ACPI_DMAR_PCI_PATH *path;
+
+ for (i = 0; i < drhd_num; i++) {
+ drhd = drhds[i];
+
+ if (VTD_DRHD_INCLUDE_PCI_ALL(drhd->Flags)) {
+ /*
+* From Intel VT-d arch spec, version 3.0:
+* If a DRHD structure with INCLUDE_PCI_ALL flag Set is
reported
+* for a Segment, it must be enumerated by BIOS after
all other
+* DRHD structures for the same Segment.
+*/
+ vtdmap = vtdmaps[i];
+ return(vtdmap);
+ }
+
+ end = (char *)drhd + drhd->Header.Length;
+ remaining = drhd->Header.Length -
sizeof(ACPI_DMAR_HARDWARE_UNIT);
+ while (remaining > sizeof(ACPI_DMAR_DEVICE_SCOPE)) {
+ device_scope = (ACPI_DMAR_DEVICE_SCOPE *)(end -
remaining);
+ remaining -= device_scope->Length;
+
+ switch (device_scope->EntryType){
+ /* 0x01 and 0x02 are PCI device entries */
+ case 0x01:
+ case 0x02:
+ break;
+ default:
