Add support for the AMD IOMMU Performance Optimization (PerfOpt) feature as defined in the AMD I/O Virtualization Technology (IOMMU) Specification, Section 3.4.9 (MMIO Offset 016Ch).
This feature allows privileged integrated I/O devices (GPUs) to bypass the IOMMU when directly accessing system memory. The IOMMU only enforces the IR/IW permission bits without GPA->SPA translations. amd_iommu_enable_perfopt() performs a detach/reattach cycle to rehome devices already on the identity domain with ATS/PRI/PASID/GCR3 disabled (skip_caps path). amd_iommu_disable_perfopt() restores those capabilities. The per-device dev_data->perfopt flag tracks state. PERF_OPT_EN is a single control bit per IOMMU, shared by every device behind that IOMMU, while enablement is requested per device. It is therefore reference counted (amd_iommu->perfopt_refcount): armed on the first requesting device and cleared on the last, so one device's teardown never clears the bit while a peer behind the same IOMMU still needs it. The per-device flag is cleared on every teardown path (blocked_domain_attach, release_device, and amd_iommu_disable_perfopt), dropping the reference with it, so a reused dev_data never carries stale PerfOpt state onto its next bind. On suspend/resume the hardware is reprogrammed from scratch: amd_iommu_perfopt_clear() forces the bit off without touching the reference count, and amd_iommu_perfopt_restore() re-asserts it from the count after early_enable_iommu(), so armed devices keep the optimization across resume without relying on each consumer driver to re-arm. The exported amd_iommu_enable_perfopt()/amd_iommu_disable_perfopt() run only from a consumer driver's bind/unbind path. group->mutex is not exposed to drivers, but a device bound to its native driver cannot have its IOMMU domain changed concurrently by the core, which serializes the detach/attach pair against core-driven attach. PerfOpt is opt-in -- only enabled when explicitly requested by a driver. Co-developed-by: Jatin Kataria <[email protected]> Signed-off-by: Jatin Kataria <[email protected]> Signed-off-by: Mario Limonciello <[email protected]> --- drivers/iommu/amd/amd_iommu.h | 3 + drivers/iommu/amd/amd_iommu_types.h | 7 + drivers/iommu/amd/init.c | 43 ++++++ drivers/iommu/amd/iommu.c | 214 ++++++++++++++++++++++++++++ include/linux/amd-iommu.h | 11 ++ 5 files changed, 278 insertions(+) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index a2fe804b038b6..1f8f9df8e6c24 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -48,6 +48,9 @@ extern u8 amd_iommu_hpt_vasize; extern unsigned long amd_iommu_pgsize_bitmap; extern bool amd_iommu_hatdis; +int amd_iommu_perfopt_clear(struct amd_iommu *iommu); +int amd_iommu_perfopt_restore(struct amd_iommu *iommu); + /* Protection domain ops */ void amd_iommu_init_identity_domain(void); struct protection_domain *protection_domain_alloc(void); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 3dbe20023456b..755421e5cd757 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -65,6 +65,7 @@ #define MMIO_MSI_ADDR_LO_OFFSET 0x015C #define MMIO_MSI_ADDR_HI_OFFSET 0x0160 #define MMIO_MSI_DATA_OFFSET 0x0164 +#define MMIO_PERF_OPT_OFFSET 0x016C #define MMIO_INTCAPXT_EVT_OFFSET 0x0170 #define MMIO_INTCAPXT_PPR_OFFSET 0x0178 #define MMIO_INTCAPXT_GALOG_OFFSET 0x0180 @@ -99,6 +100,8 @@ #define FEATURE_GLX GENMASK_ULL(15, 14) #define FEATURE_GAM_VAPIC BIT_ULL(21) #define FEATURE_PASMAX GENMASK_ULL(36, 32) +#define FEATURE_PERF_OPT BIT_ULL(45) +#define PERF_OPT_EN BIT(13) #define FEATURE_GIOSUP BIT_ULL(48) #define FEATURE_HASUP BIT_ULL(49) #define FEATURE_EPHSUP BIT_ULL(50) @@ -670,6 +673,9 @@ struct amd_iommu { /* Extended features 2 */ u64 features2; + /* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */ + int perfopt_refcount; + /* PCI device id of the IOMMU device */ u16 devid; @@ -831,6 +837,7 @@ struct iommu_dev_data { u8 ppr :1; /* Enable device PPR support */ bool use_vapic; /* Enable device to use vapic mode */ bool defer_attach; + bool perfopt; struct ratelimit_state rs; /* Ratelimit IOPF messages */ }; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 40726dfef2733..ddcf56f101675 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1942,6 +1942,9 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, if (!iommu->mmio_base) return -ENOMEM; + if (amd_iommu_perfopt_clear(iommu)) + pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index); + return init_iommu_from_acpi(iommu, h); } @@ -3032,10 +3035,46 @@ static void enable_iommus_vapic(void) #endif } +static int clear_perfopt_all(void) +{ + struct amd_iommu *iommu; + int err, ret = 0; + + for_each_iommu(iommu) { + err = amd_iommu_perfopt_clear(iommu); + if (err) + ret = err; + } + + return ret; +} + +static int restore_perfopt_all(void) +{ + struct amd_iommu *iommu; + int err, ret = 0; + + for_each_iommu(iommu) { + err = amd_iommu_perfopt_restore(iommu); + if (err) + ret = err; + } + + return ret; +} + static void disable_iommus(void) { struct amd_iommu *iommu; + /* + * PerfOpt is an optional performance bit, so a failure to clear it must + * not skip the mandatory disable below. This also runs from the void + * amd_iommu_disable() shutdown/kexec path, which cannot report an error. + */ + if (clear_perfopt_all()) + pr_err("Failed to clear PerfOpt while disabling IOMMUs\n"); + for_each_iommu(iommu) iommu_disable(iommu); @@ -3061,6 +3100,10 @@ static void amd_iommu_resume(void *data) for_each_iommu(iommu) early_enable_iommu(iommu); + /* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */ + if (restore_perfopt_all()) + pr_err("Failed to restore PerfOpt after IOMMU resume\n"); + iommu_enable_event_buffer(); amd_iommu_enable_interrupts(); } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 4dc306a4b5c62..fa60affdfc035 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2395,6 +2395,9 @@ static int attach_device(struct device *dev, if (ret) goto out; + if (dev_data->perfopt) + goto skip_caps; + /* Setup GCR3 table */ if (pdom_is_sva_capable(domain)) { ret = init_gcr3_table(dev_data, domain); @@ -2419,6 +2422,7 @@ static int attach_device(struct device *dev, pdev_enable_cap_ats(pdev); } +skip_caps: /* Update data structures */ dev_data->domain = domain; spin_lock_irqsave(&domain->lock, flags); @@ -2487,6 +2491,192 @@ static void detach_device(struct device *dev) mutex_unlock(&dev_data->mutex); } +/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */ +static int __perfopt_write(struct amd_iommu *iommu, bool enable) +{ + u32 old, val, readback; + + if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT)) + return enable ? -ENODEV : 0; + + old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (old == U32_MAX) + return -EIO; + + val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN; + if (val != old) + writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (readback == U32_MAX || + (readback & PERF_OPT_EN) != (val & PERF_OPT_EN)) + return -EIO; + return 0; +} + +/* + * PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is + * reference counted: armed on the first requesting device, cleared on the last. + */ +static int perfopt_get(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret = 0; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + if (iommu->perfopt_refcount == 0) { + ret = __perfopt_write(iommu, true); + if (ret) + goto out; + } + iommu->perfopt_refcount++; +out: + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +static int perfopt_put(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret = 0; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0) + ret = __perfopt_write(iommu, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +/* + * Force PERF_OPT_EN off without touching the refcount (used on init, shutdown, + * and suspend). The count is preserved so amd_iommu_perfopt_restore() can + * re-arm on resume. + */ +int amd_iommu_perfopt_clear(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __perfopt_write(iommu, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +/* + * Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on + * resume, so devices armed before suspend keep the optimization without each + * consumer driver re-arming. + */ +int amd_iommu_perfopt_restore(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +int amd_iommu_enable_perfopt(struct pci_dev *pdev) +{ + struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); + struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); + struct protection_domain *domain; + int ret; + + if (!iommu || !dev_data) + return -ENODEV; + + if (!(iommu->features & FEATURE_PERF_OPT)) + return -ENODEV; + + domain = dev_data->domain; + if (!domain) + return -ENODEV; + + /* Already armed for this device (e.g. re-entry on resume). */ + if (dev_data->perfopt) + return 0; + + /* + * The bit is only architecturally valid while the device is untranslated: + * identity domain with ATS/PRI/PASID off. The identity domain is + * SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3 + * table. Re-home the device onto the same identity domain with + * perfopt set, so the attach_device() skip_caps path leaves + * ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach + * pattern used by amd_iommu_attach_device(). + * + * Locking: this and amd_iommu_disable_perfopt() run only from the + * consumer driver's bind/unbind path. group->mutex is not exposed to + * drivers, but a device bound to its native driver cannot have its domain + * changed concurrently by the core (VFIO ownership is mutually exclusive; + * sysfs domain changes require an unused group), so the detach/attach pair + * is serialized without it. + */ + dev_data->perfopt = true; + detach_device(&pdev->dev); + ret = attach_device(&pdev->dev, domain); + if (ret) + goto err_restore; + + ret = perfopt_get(iommu); + if (ret) + goto err_rearm; + + dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index); + return 0; + +err_rearm: + detach_device(&pdev->dev); +err_restore: + dev_data->perfopt = false; + if (attach_device(&pdev->dev, domain)) + pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n"); + dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n", + iommu->index, ret); + return ret; +} +EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt); + +void amd_iommu_disable_perfopt(struct pci_dev *pdev) +{ + struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); + struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); + struct protection_domain *domain; + + if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain) + return; + + if (WARN_ON(perfopt_put(iommu))) + pci_err(pdev, "failed to clear PerfOpt\n"); + + /* + * Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its + * identity domain with the flag cleared, so a later bind without PerfOpt + * sees a normally-capable device. See the locking note in + * amd_iommu_enable_perfopt(). + */ + domain = dev_data->domain; + dev_data->perfopt = false; + detach_device(&pdev->dev); + if (attach_device(&pdev->dev, domain)) + pci_err(pdev, "failed to restore caps after PerfOpt disable\n"); +} +EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt); static struct iommu_device *amd_iommu_probe_device(struct device *dev) { struct iommu_device *iommu_dev; @@ -2554,6 +2744,14 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) static void amd_iommu_release_device(struct device *dev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + + if (dev_data->perfopt) { + if (WARN_ON(perfopt_put(iommu))) + dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n", + iommu->index); + dev_data->perfopt = false; + } WARN_ON(dev_data->domain); @@ -2928,6 +3126,19 @@ static int blocked_domain_attach_device(struct iommu_domain *domain, struct iommu_domain *old) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + + /* + * blocked_domain is also the .release_domain, so this is the normal + * teardown path: drop the reference and clear the flag here too, and + * don't fail teardown if the WARN-guarded write doesn't stick. + */ + if (dev_data->perfopt) { + if (WARN_ON(perfopt_put(iommu))) + dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n", + iommu->index); + dev_data->perfopt = false; + } if (dev_data->domain) detach_device(dev); @@ -2996,6 +3207,9 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); int ret; + if (dev_data->perfopt && !pdom_is_in_pt_mode(domain)) + return -EBUSY; + /* * Skip attach device to domain if new domain is same as * devices current domain diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index edcee9f5335a6..e03575cbc08c6 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -76,4 +76,15 @@ static inline int amd_iommu_snp_disable(void) { return 0; } static inline bool amd_iommu_sev_tio_supported(void) { return false; } #endif +#ifdef CONFIG_AMD_IOMMU +int amd_iommu_enable_perfopt(struct pci_dev *pdev); +void amd_iommu_disable_perfopt(struct pci_dev *pdev); +#else +static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev) +{ + return 0; +} +static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { } +#endif + #endif /* _ASM_X86_AMD_IOMMU_H */ -- 2.43.0
