On Thu Aug 27 15:33:05 2026 +0300, Antti Laakso wrote:
> We are about to add support for ipu7 mmu. Split ipu6 mmu
> hardware specific and common code to separate files.
>
> Signed-off-by: Antti Laakso <[email protected]>
> Signed-off-by: Sakari Ailus <[email protected]>
Patch committed.
Thanks,
Sakari Ailus
drivers/media/pci/intel/ipu6/Makefile | 1 +
drivers/media/pci/intel/ipu6/ipu6-dma.c | 4 +-
drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c | 296 +++++++++++++++++++++++++++++
drivers/media/pci/intel/ipu6/ipu6-mmu.c | 120 ++----------
drivers/media/pci/intel/ipu6/ipu6-mmu.h | 149 ++++++++++++++-
drivers/media/pci/intel/ipu6/ipu6.c | 125 +-----------
drivers/media/pci/intel/ipu6/ipu6.h | 132 +------------
7 files changed, 456 insertions(+), 371 deletions(-)
---
diff --git a/drivers/media/pci/intel/ipu6/Makefile
b/drivers/media/pci/intel/ipu6/Makefile
index a821b0a1567f..2106ab84acde 100644
--- a/drivers/media/pci/intel/ipu6/Makefile
+++ b/drivers/media/pci/intel/ipu6/Makefile
@@ -4,6 +4,7 @@ intel-ipu6-y := ipu6.o \
ipu6-bus.o \
ipu6-dma.o \
ipu6-mmu.o \
+ ipu6-mmu-hw.o \
ipu6-buttress.o \
ipu6-cpd.o \
ipu6-fw-com.o
diff --git a/drivers/media/pci/intel/ipu6/ipu6-dma.c
b/drivers/media/pci/intel/ipu6/ipu6-dma.c
index fdcdb15b073c..6e8295d9c3a3 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-dma.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-dma.c
@@ -286,7 +286,7 @@ void ipu6_dma_free(struct ipu6_bus_device *sys, size_t
size, void *vaddr,
__free_buffer(pages, size, attrs);
- mmu->tlb_invalidate(mmu);
+ mmu->ops->tlb_invalidate(mmu);
__free_iova(&mmu->dmap->iovad, iova);
@@ -366,7 +366,7 @@ void ipu6_dma_unmap_sg(struct ipu6_bus_device *sys, struct
scatterlist *sglist,
ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
PFN_PHYS(iova_size(iova)));
- mmu->tlb_invalidate(mmu);
+ mmu->ops->tlb_invalidate(mmu);
__free_iova(&mmu->dmap->iovad, iova);
}
EXPORT_SYMBOL_NS_GPL(ipu6_dma_unmap_sg, "INTEL_IPU6");
diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c
b/drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c
new file mode 100644
index 000000000000..2b395fbd5969
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Intel Corporation
+ */
+#include <asm/barrier.h>
+
+#include <linux/bits.h>
+#include <linux/gfp.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+#include "ipu6.h"
+#include "ipu6-dma.h"
+#include "ipu6-mmu.h"
+#include "ipu6-platform-regs.h"
+
+#define ISP_PAGE_SHIFT 12
+#define ISP_PAGE_SIZE BIT(ISP_PAGE_SHIFT)
+#define ISP_PAGE_MASK (~(ISP_PAGE_SIZE - 1))
+
+#define ISP_L1PT_SHIFT 22
+#define ISP_L1PT_MASK (~((1U << ISP_L1PT_SHIFT) - 1))
+
+#define ISP_L2PT_SHIFT 12
+#define ISP_L2PT_MASK (~(ISP_L1PT_MASK | (~(ISP_PAGE_MASK))))
+
+#define ISP_L1PT_PTES 1024
+#define ISP_L2PT_PTES 1024
+
+#define ISP_PADDR_SHIFT 12
+
+#define REG_TLB_INVALIDATE 0x0000
+
+#define REG_L1_PHYS 0x0004 /* 27-bit pfn */
+#define REG_INFO 0x0008
+
+#define TBL_PHYS_ADDR(a) ((phys_addr_t)(a) << ISP_PADDR_SHIFT)
+
+static struct ipu6_mmu_hw ipu6_isys_mmu_hwdata[] = {
+ {
+ .offset = IPU6_ISYS_IOMMU0_OFFSET,
+ .info_bits = IPU6_INFO_REQUEST_DESTINATION_IOSF,
+ .nr_l1streams = 16,
+ .l1_block_sz = {
+ 3, 8, 2, 2, 2, 2, 2, 2, 1, 1,
+ 1, 1, 1, 1, 1, 1
+ },
+ .nr_l2streams = 16,
+ .l2_block_sz = {
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2
+ },
+ .insert_read_before_invalidate = false,
+ .l1_stream_id_reg_offset =
+ IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+ .l2_stream_id_reg_offset =
+ IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
+ },
+ {
+ .offset = IPU6_ISYS_IOMMU1_OFFSET,
+ .info_bits = 0,
+ .nr_l1streams = 16,
+ .l1_block_sz = {
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 1, 1, 4
+ },
+ .nr_l2streams = 16,
+ .l2_block_sz = {
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2
+ },
+ .insert_read_before_invalidate = false,
+ .l1_stream_id_reg_offset =
+ IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+ .l2_stream_id_reg_offset =
+ IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
+ },
+ {
+ .offset = IPU6_ISYS_IOMMUI_OFFSET,
+ .info_bits = 0,
+ .nr_l1streams = 0,
+ .nr_l2streams = 0,
+ .insert_read_before_invalidate = false,
+ },
+};
+
+static struct ipu6_mmu_hw ipu6_psys_mmu_hwdata[] = {
+ {
+ .offset = IPU6_PSYS_IOMMU0_OFFSET,
+ .info_bits =
+ IPU6_INFO_REQUEST_DESTINATION_IOSF,
+ .nr_l1streams = 16,
+ .l1_block_sz = {
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2
+ },
+ .nr_l2streams = 16,
+ .l2_block_sz = {
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2
+ },
+ .insert_read_before_invalidate = false,
+ .l1_stream_id_reg_offset =
+ IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+ .l2_stream_id_reg_offset =
+ IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
+ },
+ {
+ .offset = IPU6_PSYS_IOMMU1_OFFSET,
+ .info_bits = 0,
+ .nr_l1streams = 32,
+ .l1_block_sz = {
+ 1, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 10,
+ 5, 4, 14, 6, 4, 14, 6, 4, 8,
+ 4, 2, 1, 1, 1, 1, 14
+ },
+ .nr_l2streams = 32,
+ .l2_block_sz = {
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2
+ },
+ .insert_read_before_invalidate = false,
+ .l1_stream_id_reg_offset =
+ IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+ .l2_stream_id_reg_offset =
+ IPU6_PSYS_MMU1W_L2_STREAM_ID_REG_OFFSET,
+ },
+ {
+ .offset = IPU6_PSYS_IOMMU1R_OFFSET,
+ .info_bits = 0,
+ .nr_l1streams = 16,
+ .l1_block_sz = {
+ 1, 4, 4, 4, 4, 16, 8, 4, 32,
+ 16, 16, 2, 2, 2, 1, 12
+ },
+ .nr_l2streams = 16,
+ .l2_block_sz = {
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2
+ },
+ .insert_read_before_invalidate = false,
+ .l1_stream_id_reg_offset =
+ IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+ .l2_stream_id_reg_offset =
+ IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
+ },
+ {
+ .offset = IPU6_PSYS_IOMMUI_OFFSET,
+ .info_bits = 0,
+ .nr_l1streams = 0,
+ .nr_l2streams = 0,
+ .insert_read_before_invalidate = false,
+ },
+};
+
+struct ipu6_mmu_hwdata {
+ struct ipu6_mmu_hw *hwdata;
+ unsigned int nr_mmus;
+};
+
+static const struct ipu6_mmu_hwdata ipu6_mmu_hwdata_lookup[IPU_SUBSYS_NUM] = {
+ [IPU_PSYS] = {
+ .hwdata = ipu6_psys_mmu_hwdata,
+ .nr_mmus = ARRAY_SIZE(ipu6_psys_mmu_hwdata),
+ },
+ [IPU_ISYS] = {
+ .hwdata = ipu6_isys_mmu_hwdata,
+ .nr_mmus = ARRAY_SIZE(ipu6_isys_mmu_hwdata),
+ },
+};
+
+static void __ipu6_tlb_invalidate(struct ipu6_mmu *mmu)
+{
+ struct ipu6_mmu_hw *mmu_hw = mmu->ipu6_mmu_hw;
+ unsigned long flags;
+ unsigned int i;
+
+ spin_lock_irqsave(&mmu->ready_lock, flags);
+ if (!mmu->ready) {
+ spin_unlock_irqrestore(&mmu->ready_lock, flags);
+ return;
+ }
+
+ for (i = 0; i < mmu->nr_mmus; i++) {
+ /*
+ * To avoid the HW bug induced dead lock in some of the IPU6
+ * MMUs on successive invalidate calls, we need to first do a
+ * read to the page table base before writing the invalidate
+ * register. MMUs which need to implement this WA, will have
+ * the insert_read_before_invalidate flags set as true.
+ * Disregard the return value of the read.
+ */
+ if (mmu_hw[i].insert_read_before_invalidate)
+ readl(mmu_hw[i].base + REG_L1_PHYS);
+
+ writel(0xffffffff, mmu_hw[i].base + REG_TLB_INVALIDATE);
+ /*
+ * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
+ * When the actual MMIO write reaches the IPU6 TLB Invalidate
+ * register, wmb() will force the TLB invalidate out if the CPU
+ * attempts to update the IOMMU page table (or sooner).
+ */
+ wmb();
+ }
+ spin_unlock_irqrestore(&mmu->ready_lock, flags);
+}
+
+static int __ipu6_mmu_hw_init(struct ipu6_mmu *mmu)
+{
+ struct ipu6_mmu_info *mmu_info = mmu->dmap->mmu_info;
+ struct ipu6_mmu_hw *mmu_hw = mmu->ipu6_mmu_hw;
+
+ /* Initialise the each MMU HW block */
+ for (unsigned int i = 0; i < mmu->nr_mmus; i++) {
+ unsigned int j;
+ u16 block_addr;
+
+ /* Write page table address per MMU */
+ writel((phys_addr_t)mmu_info->l1_pt_dma,
+ mmu_hw[i].base + REG_L1_PHYS);
+
+ /* Set info bits per MMU */
+ writel(mmu_hw[i].info_bits, mmu_hw[i].base + REG_INFO);
+
+ /* Configure MMU TLB stream configuration for L1 */
+ for (j = 0, block_addr = 0; j < mmu_hw[i].nr_l1streams;
+ block_addr += mmu_hw[i].l1_block_sz[j], j++) {
+ if (block_addr > IPU6_MAX_LI_BLOCK_ADDR) {
+ dev_err(mmu->dev, "invalid L1 configuration\n");
+ return -EINVAL;
+ }
+
+ /* Write block start address for each streams */
+ writel(block_addr, mmu_hw[i].base +
+ mmu_hw[i].l1_stream_id_reg_offset + 4 * j);
+ }
+
+ /* Configure MMU TLB stream configuration for L2 */
+ for (j = 0, block_addr = 0; j < mmu_hw[i].nr_l2streams;
+ block_addr += mmu_hw[i].l2_block_sz[j], j++) {
+ if (block_addr > IPU6_MAX_L2_BLOCK_ADDR) {
+ dev_err(mmu->dev, "invalid L2 configuration\n");
+ return -EINVAL;
+ }
+
+ writel(block_addr, mmu_hw[i].base +
+ mmu_hw[i].l2_stream_id_reg_offset + 4 * j);
+ }
+ }
+
+ return 0;
+}
+
+static int __ipu6_mmu_init_hw_data(struct ipu6_mmu *mmu, struct device *dev,
+ void __iomem *base)
+{
+ const struct ipu6_mmu_hwdata *lookup;
+ struct ipu6_mmu_hw *mmu_hw, *src;
+ unsigned int i, nr_mmus;
+
+ if (mmu->mmid >= IPU_SUBSYS_NUM)
+ return -EINVAL;
+
+ lookup = &ipu6_mmu_hwdata_lookup[mmu->mmid];
+ src = lookup->hwdata;
+ nr_mmus = lookup->nr_mmus;
+
+ mmu_hw = devm_kcalloc(dev, nr_mmus, sizeof(*mmu_hw), GFP_KERNEL);
+ if (!mmu_hw)
+ return -ENOMEM;
+
+ for (i = 0; i < nr_mmus; i++) {
+ if (src[i].nr_l1streams > IPU6_MMU_MAX_TLB_L1_STREAMS ||
+ src[i].nr_l2streams > IPU6_MMU_MAX_TLB_L2_STREAMS)
+ return -EINVAL;
+
+ mmu_hw[i] = src[i];
+ mmu_hw[i].base = base + src[i].offset;
+ }
+
+ mmu->nr_mmus = nr_mmus;
+ mmu->ipu6_mmu_hw = mmu_hw;
+
+ return 0;
+}
+
+const struct ipu6_mmu_hw_ops ipu6_mmu_ops = {
+ .init_hw_data = __ipu6_mmu_init_hw_data,
+ .hw_init = __ipu6_mmu_hw_init,
+ .tlb_invalidate = __ipu6_tlb_invalidate,
+};
diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu.c
b/drivers/media/pci/intel/ipu6/ipu6-mmu.c
index 6b1a49ebbcc2..98ac59d00b78 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-mmu.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-mmu.c
@@ -5,7 +5,6 @@
#include <asm/barrier.h>
#include <linux/align.h>
-#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/bits.h>
#include <linux/bug.h>
@@ -13,10 +12,8 @@
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/gfp.h>
-#include <linux/io.h>
#include <linux/iova.h>
#include <linux/math.h>
-#include <linux/minmax.h>
#include <linux/mm.h>
#include <linux/pfn.h>
#include <linux/slab.h>
@@ -51,42 +48,6 @@
#define TBL_PHYS_ADDR(a) ((phys_addr_t)(a) << ISP_PADDR_SHIFT)
-static void tlb_invalidate(struct ipu6_mmu *mmu)
-{
- unsigned long flags;
- unsigned int i;
-
- spin_lock_irqsave(&mmu->ready_lock, flags);
- if (!mmu->ready) {
- spin_unlock_irqrestore(&mmu->ready_lock, flags);
- return;
- }
-
- for (i = 0; i < mmu->nr_mmus; i++) {
- /*
- * To avoid the HW bug induced dead lock in some of the IPU6
- * MMUs on successive invalidate calls, we need to first do a
- * read to the page table base before writing the invalidate
- * register. MMUs which need to implement this WA, will have
- * the insert_read_before_invalidate flags set as true.
- * Disregard the return value of the read.
- */
- if (mmu->mmu_hw[i].insert_read_before_invalidate)
- readl(mmu->mmu_hw[i].base + REG_L1_PHYS);
-
- writel(0xffffffff, mmu->mmu_hw[i].base +
- REG_TLB_INVALIDATE);
- /*
- * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
- * When the actual MMIO write reaches the IPU6 TLB Invalidate
- * register, wmb() will force the TLB invalidate out if the CPU
- * attempts to update the IOMMU page table (or sooner).
- */
- wmb();
- }
- spin_unlock_irqrestore(&mmu->ready_lock, flags);
-}
-
#ifdef DEBUG
static void page_table_dump(struct ipu6_mmu_info *mmu_info)
{
@@ -472,55 +433,14 @@ out_free_iova:
int ipu6_mmu_hw_init(struct ipu6_mmu *mmu)
{
- struct ipu6_mmu_info *mmu_info;
unsigned long flags;
- unsigned int i;
-
- mmu_info = mmu->dmap->mmu_info;
-
- /* Initialise the each MMU HW block */
- for (i = 0; i < mmu->nr_mmus; i++) {
- struct ipu6_mmu_hw *mmu_hw = &mmu->mmu_hw[i];
- unsigned int j;
- u16 block_addr;
-
- /* Write page table address per MMU */
- writel((phys_addr_t)mmu_info->l1_pt_dma,
- mmu->mmu_hw[i].base + REG_L1_PHYS);
-
- /* Set info bits per MMU */
- writel(mmu->mmu_hw[i].info_bits,
- mmu->mmu_hw[i].base + REG_INFO);
-
- /* Configure MMU TLB stream configuration for L1 */
- for (j = 0, block_addr = 0; j < mmu_hw->nr_l1streams;
- block_addr += mmu->mmu_hw[i].l1_block_sz[j], j++) {
- if (block_addr > IPU6_MAX_LI_BLOCK_ADDR) {
- dev_err(mmu->dev, "invalid L1 configuration\n");
- return -EINVAL;
- }
-
- /* Write block start address for each streams */
- writel(block_addr, mmu_hw->base +
- mmu_hw->l1_stream_id_reg_offset + 4 * j);
- }
-
- /* Configure MMU TLB stream configuration for L2 */
- for (j = 0, block_addr = 0; j < mmu_hw->nr_l2streams;
- block_addr += mmu->mmu_hw[i].l2_block_sz[j], j++) {
- if (block_addr > IPU6_MAX_L2_BLOCK_ADDR) {
- dev_err(mmu->dev, "invalid L2 configuration\n");
- return -EINVAL;
- }
+ int ret;
- writel(block_addr, mmu_hw->base +
- mmu_hw->l2_stream_id_reg_offset + 4 * j);
- }
- }
+ ret = mmu->ops->hw_init(mmu);
+ if (ret)
+ return ret;
if (!mmu->trash_page) {
- int ret;
-
mmu->trash_page = alloc_page(GFP_KERNEL);
if (!mmu->trash_page) {
dev_err(mmu->dev, "insufficient memory for trash
buffer\n");
@@ -746,44 +666,26 @@ static void ipu6_mmu_destroy(struct ipu6_mmu *mmu)
}
struct ipu6_mmu *ipu6_mmu_init(struct device *dev,
- void __iomem *base, int mmid,
- const struct ipu6_hw_variants *hw)
+ void __iomem *base, int mmid)
{
struct ipu6_device *isp = pci_get_drvdata(to_pci_dev(dev));
- struct ipu6_mmu_hw *mmu_hw;
struct ipu6_mmu *mmu;
- unsigned int i;
-
- if (hw->nr_mmus > IPU6_MMU_MAX_DEVICES)
- return ERR_PTR(-EINVAL);
-
- mmu_hw = devm_kcalloc(dev, sizeof(*mmu_hw), hw->nr_mmus, GFP_KERNEL);
- if (!mmu_hw)
- return ERR_PTR(-ENOMEM);
-
- for (i = 0; i < hw->nr_mmus; i++) {
- const struct ipu6_mmu_hw *src_mmu = &hw->mmu_hw[i];
-
- if (src_mmu->nr_l1streams > IPU6_MMU_MAX_TLB_L1_STREAMS ||
- src_mmu->nr_l2streams > IPU6_MMU_MAX_TLB_L2_STREAMS)
- return ERR_PTR(-EINVAL);
-
- mmu_hw[i] = *src_mmu;
- mmu_hw[i].base = base + src_mmu->offset;
- }
+ int ret;
mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL);
if (!mmu)
return ERR_PTR(-ENOMEM);
+ mmu->ops = &ipu6_mmu_ops;
mmu->mmid = mmid;
- mmu->mmu_hw = mmu_hw;
- mmu->nr_mmus = hw->nr_mmus;
- mmu->tlb_invalidate = tlb_invalidate;
mmu->ready = false;
INIT_LIST_HEAD(&mmu->vma_list);
spin_lock_init(&mmu->ready_lock);
+ ret = mmu->ops->init_hw_data(mmu, dev, base);
+ if (ret)
+ return ERR_PTR(ret);
+
mmu->dmap = alloc_dma_mapping(isp);
if (!mmu->dmap) {
dev_err(dev, "can't alloc dma mapping\n");
diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu.h
b/drivers/media/pci/intel/ipu6/ipu6-mmu.h
index 35cac80d2d4e..a741f1c0e284 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-mmu.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-mmu.h
@@ -1,5 +1,5 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2013--2024 Intel Corporation */
+/* Copyright (C) 2013--2026 Intel Corporation */
#ifndef IPU6_MMU_H
#define IPU6_MMU_H
@@ -10,7 +10,6 @@
struct device;
struct page;
-struct ipu6_hw_variants;
struct ipu6_mmu_info {
struct device *dev;
@@ -32,10 +31,147 @@ struct ipu6_mmu_info {
struct ipu6_dma_mapping *dmap;
};
+/*
+ * MMU Invalidation HW bug workaround by ZLW mechanism
+ *
+ * Old IPU6 MMUV2 has a bug in the invalidation mechanism which might result in
+ * wrong translation or replication of the translation. This will cause data
+ * corruption. So we cannot directly use the MMU V2 invalidation registers
+ * to invalidate the MMU. Instead, whenever an invalidate is called, we need to
+ * clear the TLB by evicting all the valid translations by filling it with
trash
+ * buffer (which is guaranteed not to be used by any other processes). ZLW is
+ * used to fill the L1 and L2 caches with the trash buffer translations. ZLW
+ * or Zero length write, is pre-fetch mechanism to pre-fetch the pages in
+ * advance to the L1 and L2 caches without triggering any memory operations.
+ *
+ * In MMU V2, L1 -> 16 streams and 64 blocks, maximum 16 blocks per stream
+ * One L1 block has 16 entries, hence points to 16 * 4K pages
+ * L2 -> 16 streams and 32 blocks. 2 blocks per streams
+ * One L2 block maps to 1024 L1 entries, hence points to 4MB address range
+ * 2 blocks per L2 stream means, 1 stream points to 8MB range
+ *
+ * As we need to clear the caches and 8MB being the biggest cache size, we need
+ * to have trash buffer which points to 8MB address range. As these trash
+ * buffers are not used for any memory transactions, we need only the least
+ * amount of physical memory. So we reserve 8MB IOVA address range but only
+ * one page is reserved from physical memory. Each of this 8MB IOVA address
+ * range is then mapped to the same physical memory page.
+ */
+/* One L2 entry maps 1024 L1 entries and one L1 entry per page */
+#define IPU6_MMUV2_L2_RANGE (1024 * PAGE_SIZE)
+/* Max L2 blocks per stream */
+#define IPU6_MMUV2_MAX_L2_BLOCKS 2
+/* Max L1 blocks per stream */
+#define IPU6_MMUV2_MAX_L1_BLOCKS 16
+#define IPU6_MMUV2_TRASH_RANGE (IPU6_MMUV2_L2_RANGE * IPU6_MMUV2_MAX_L2_BLOCKS)
+/* Entries per L1 block */
+#define MMUV2_ENTRIES_PER_L1_BLOCK 16
+#define MMUV2_TRASH_L1_BLOCK_OFFSET (MMUV2_ENTRIES_PER_L1_BLOCK * PAGE_SIZE)
+#define MMUV2_TRASH_L2_BLOCK_OFFSET IPU6_MMUV2_L2_RANGE
+
+/*
+ * In some of the IPU6 MMUs, there is provision to configure L1 and L2 page
+ * table caches. Both these L1 and L2 caches are divided into multiple sections
+ * called streams. There is maximum 16 streams for both caches. Each of these
+ * sections are subdivided into multiple blocks. When nr_l1streams = 0 and
+ * nr_l2streams = 0, means the MMU is of type MMU_V1 and do not support
+ * L1/L2 page table caches.
+ *
+ * L1 stream per block sizes are configurable and varies per usecase.
+ * L2 has constant block sizes - 2 blocks per stream.
+ *
+ * MMU1 support pre-fetching of the pages to have less cache lookup misses. To
+ * enable the pre-fetching, MMU1 AT (Address Translator) device registers
+ * need to be configured.
+ *
+ * There are four types of memory accesses which requires ZLW configuration.
+ * ZLW(Zero Length Write) is a mechanism to enable VT-d pre-fetching on IOMMU.
+ *
+ * 1. Sequential Access or 1D mode
+ * Set ZLW_EN -> 1
+ * set ZLW_PAGE_CROSS_1D -> 1
+ * Set ZLW_N to "N" pages so that ZLW will be inserte N pages ahead where
+ * N is pre-defined and hardcoded in the platform data
+ * Set ZLW_2D -> 0
+ *
+ * 2. ZLW 2D mode
+ * Set ZLW_EN -> 1
+ * set ZLW_PAGE_CROSS_1D -> 1,
+ * Set ZLW_N -> 0
+ * Set ZLW_2D -> 1
+ *
+ * 3. ZLW Enable (no 1D or 2D mode)
+ * Set ZLW_EN -> 1
+ * set ZLW_PAGE_CROSS_1D -> 0,
+ * Set ZLW_N -> 0
+ * Set ZLW_2D -> 0
+ *
+ * 4. ZLW disable
+ * Set ZLW_EN -> 0
+ * set ZLW_PAGE_CROSS_1D -> 0,
+ * Set ZLW_N -> 0
+ * Set ZLW_2D -> 0
+ *
+ * To configure the ZLW for the above memory access, four registers are
+ * available. Hence to track these four settings, we have the following entries
+ * in the struct ipu6_mmu_hw. Each of these entries are per stream and
+ * available only for the L1 streams.
+ *
+ * a. l1_zlw_en -> To track zlw enabled per stream (ZLW_EN)
+ * b. l1_zlw_1d_mode -> Track 1D mode per stream. ZLW inserted at page boundary
+ * c. l1_ins_zlw_ahead_pages -> to track how advance the ZLW need to be
inserted
+ * Insert ZLW request N pages ahead address.
+ * d. l1_zlw_2d_mode -> To track 2D mode per stream (ZLW_2D)
+ *
+ *
+ * Currently L1/L2 streams, blocks, AT ZLW configurations etc. are pre-defined
+ * as per the usecase specific calculations. Any change to this pre-defined
+ * table has to happen in sync with IPU6 FW.
+ */
+
+struct ipu6_mmu_hw {
+ union {
+ unsigned long offset;
+ void __iomem *base;
+ };
+ u32 info_bits;
+ u8 nr_l1streams;
+ /*
+ * L1 has variable blocks per stream - total of 64 blocks and maximum of
+ * 16 blocks per stream. Configurable by using the block start address
+ * per stream. Block start address is calculated from the block size
+ */
+ u8 l1_block_sz[IPU6_MMU_MAX_TLB_L1_STREAMS];
+ /* Is ZLW is enabled in each stream */
+ bool l1_zlw_en[IPU6_MMU_MAX_TLB_L1_STREAMS];
+ bool l1_zlw_1d_mode[IPU6_MMU_MAX_TLB_L1_STREAMS];
+ u8 l1_ins_zlw_ahead_pages[IPU6_MMU_MAX_TLB_L1_STREAMS];
+ bool l1_zlw_2d_mode[IPU6_MMU_MAX_TLB_L1_STREAMS];
+
+ u32 l1_stream_id_reg_offset;
+ u32 l2_stream_id_reg_offset;
+
+ u8 nr_l2streams;
+ /*
+ * L2 has fixed 2 blocks per stream. Block address is calculated
+ * from the block size
+ */
+ u8 l2_block_sz[IPU6_MMU_MAX_TLB_L2_STREAMS];
+ /* flag to track if WA is needed for successive invalidate HW bug */
+ bool insert_read_before_invalidate;
+};
+
+struct ipu6_mmu_hw_ops {
+ int (*init_hw_data)(struct ipu6_mmu *mmu, struct device *dev,
+ void __iomem *base);
+ int (*hw_init)(struct ipu6_mmu *mmu);
+ void (*tlb_invalidate)(struct ipu6_mmu *mmu);
+};
+
struct ipu6_mmu {
struct list_head node;
- struct ipu6_mmu_hw *mmu_hw;
+ struct ipu6_mmu_hw *ipu6_mmu_hw;
unsigned int nr_mmus;
unsigned int mmid;
@@ -52,12 +188,13 @@ struct ipu6_mmu {
bool ready;
spinlock_t ready_lock; /* Serialize access to bool ready */
- void (*tlb_invalidate)(struct ipu6_mmu *mmu);
+ const struct ipu6_mmu_hw_ops *ops;
};
+extern const struct ipu6_mmu_hw_ops ipu6_mmu_ops;
+
struct ipu6_mmu *ipu6_mmu_init(struct device *dev,
- void __iomem *base, int mmid,
- const struct ipu6_hw_variants *hw);
+ void __iomem *base, int mmid);
void ipu6_mmu_cleanup(struct ipu6_mmu *mmu);
int ipu6_mmu_hw_init(struct ipu6_mmu *mmu);
void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu);
diff --git a/drivers/media/pci/intel/ipu6/ipu6.c
b/drivers/media/pci/intel/ipu6/ipu6.c
index 4af0b068e759..51174cafb1b4 100644
--- a/drivers/media/pci/intel/ipu6/ipu6.c
+++ b/drivers/media/pci/intel/ipu6/ipu6.c
@@ -74,54 +74,6 @@ struct ipu6_cell_program {
static struct ipu6_isys_internal_pdata isys_ipdata = {
.hw_variant = {
.offset = IPU6_UNIFIED_OFFSET,
- .nr_mmus = 3,
- .mmu_hw = {
- {
- .offset = IPU6_ISYS_IOMMU0_OFFSET,
- .info_bits = IPU6_INFO_REQUEST_DESTINATION_IOSF,
- .nr_l1streams = 16,
- .l1_block_sz = {
- 3, 8, 2, 2, 2, 2, 2, 2, 1, 1,
- 1, 1, 1, 1, 1, 1
- },
- .nr_l2streams = 16,
- .l2_block_sz = {
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2
- },
- .insert_read_before_invalidate = false,
- .l1_stream_id_reg_offset =
- IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
- .l2_stream_id_reg_offset =
- IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
- },
- {
- .offset = IPU6_ISYS_IOMMU1_OFFSET,
- .info_bits = 0,
- .nr_l1streams = 16,
- .l1_block_sz = {
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 1, 1, 4
- },
- .nr_l2streams = 16,
- .l2_block_sz = {
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2
- },
- .insert_read_before_invalidate = false,
- .l1_stream_id_reg_offset =
- IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
- .l2_stream_id_reg_offset =
- IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
- },
- {
- .offset = IPU6_ISYS_IOMMUI_OFFSET,
- .info_bits = 0,
- .nr_l1streams = 0,
- .nr_l2streams = 0,
- .insert_read_before_invalidate = false,
- },
- },
.cdc_fifos = 3,
.cdc_fifo_threshold = {6, 8, 2},
.dmem_offset = IPU6_ISYS_DMEM_OFFSET,
@@ -133,78 +85,6 @@ static struct ipu6_isys_internal_pdata isys_ipdata = {
static struct ipu6_psys_internal_pdata psys_ipdata = {
.hw_variant = {
.offset = IPU6_UNIFIED_OFFSET,
- .nr_mmus = 4,
- .mmu_hw = {
- {
- .offset = IPU6_PSYS_IOMMU0_OFFSET,
- .info_bits =
- IPU6_INFO_REQUEST_DESTINATION_IOSF,
- .nr_l1streams = 16,
- .l1_block_sz = {
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2
- },
- .nr_l2streams = 16,
- .l2_block_sz = {
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2
- },
- .insert_read_before_invalidate = false,
- .l1_stream_id_reg_offset =
- IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
- .l2_stream_id_reg_offset =
- IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
- },
- {
- .offset = IPU6_PSYS_IOMMU1_OFFSET,
- .info_bits = 0,
- .nr_l1streams = 32,
- .l1_block_sz = {
- 1, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 10,
- 5, 4, 14, 6, 4, 14, 6, 4, 8,
- 4, 2, 1, 1, 1, 1, 14
- },
- .nr_l2streams = 32,
- .l2_block_sz = {
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2
- },
- .insert_read_before_invalidate = false,
- .l1_stream_id_reg_offset =
- IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
- .l2_stream_id_reg_offset =
- IPU6_PSYS_MMU1W_L2_STREAM_ID_REG_OFFSET,
- },
- {
- .offset = IPU6_PSYS_IOMMU1R_OFFSET,
- .info_bits = 0,
- .nr_l1streams = 16,
- .l1_block_sz = {
- 1, 4, 4, 4, 4, 16, 8, 4, 32,
- 16, 16, 2, 2, 2, 1, 12
- },
- .nr_l2streams = 16,
- .l2_block_sz = {
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2
- },
- .insert_read_before_invalidate = false,
- .l1_stream_id_reg_offset =
- IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
- .l2_stream_id_reg_offset =
- IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
- },
- {
- .offset = IPU6_PSYS_IOMMUI_OFFSET,
- .info_bits = 0,
- .nr_l1streams = 0,
- .nr_l2streams = 0,
- .insert_read_before_invalidate = false,
- },
- },
.dmem_offset = IPU6_PSYS_DMEM_OFFSET,
},
};
@@ -478,7 +358,7 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
"ipu6_bus_initialize_device isys failed\n");
}
- isys_adev->mmu = ipu6_mmu_init(dev, base, IPU_ISYS,
&ipdata->hw_variant);
+ isys_adev->mmu = ipu6_mmu_init(dev, base, IPU_ISYS);
if (IS_ERR(isys_adev->mmu)) {
put_device(&isys_adev->auxdev.dev);
return dev_err_cast_probe(dev, isys_adev->mmu,
@@ -518,8 +398,7 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
"ipu6_bus_initialize_device psys failed\n");
}
- psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, IPU_PSYS,
- &ipdata->hw_variant);
+ psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, IPU_PSYS);
if (IS_ERR(psys_adev->mmu)) {
put_device(&psys_adev->auxdev.dev);
return dev_err_cast_probe(&pdev->dev, psys_adev->mmu,
diff --git a/drivers/media/pci/intel/ipu6/ipu6.h
b/drivers/media/pci/intel/ipu6/ipu6.h
index a14b067f169f..debd571b6f6d 100644
--- a/drivers/media/pci/intel/ipu6/ipu6.h
+++ b/drivers/media/pci/intel/ipu6/ipu6.h
@@ -74,6 +74,7 @@ struct ipu6_device {
#define IPU_PSYS 0
#define IPU_ISYS 1
+#define IPU_SUBSYS_NUM 2
#define IPU6_ISYS_NAME "isys"
#define IPU6_PSYS_NAME "psys"
@@ -121,135 +122,6 @@ struct ipu6_device {
#define IPU6_BTRS_ARB_STALL_MODE_VC1 \
IPU6_BTRS_ARB_MODE_TYPE_REARB
-/*
- * MMU Invalidation HW bug workaround by ZLW mechanism
- *
- * Old IPU6 MMUV2 has a bug in the invalidation mechanism which might result in
- * wrong translation or replication of the translation. This will cause data
- * corruption. So we cannot directly use the MMU V2 invalidation registers
- * to invalidate the MMU. Instead, whenever an invalidate is called, we need to
- * clear the TLB by evicting all the valid translations by filling it with
trash
- * buffer (which is guaranteed not to be used by any other processes). ZLW is
- * used to fill the L1 and L2 caches with the trash buffer translations. ZLW
- * or Zero length write, is pre-fetch mechanism to pre-fetch the pages in
- * advance to the L1 and L2 caches without triggering any memory operations.
- *
- * In MMU V2, L1 -> 16 streams and 64 blocks, maximum 16 blocks per stream
- * One L1 block has 16 entries, hence points to 16 * 4K pages
- * L2 -> 16 streams and 32 blocks. 2 blocks per streams
- * One L2 block maps to 1024 L1 entries, hence points to 4MB address range
- * 2 blocks per L2 stream means, 1 stream points to 8MB range
- *
- * As we need to clear the caches and 8MB being the biggest cache size, we need
- * to have trash buffer which points to 8MB address range. As these trash
- * buffers are not used for any memory transactions, we need only the least
- * amount of physical memory. So we reserve 8MB IOVA address range but only
- * one page is reserved from physical memory. Each of this 8MB IOVA address
- * range is then mapped to the same physical memory page.
- */
-/* One L2 entry maps 1024 L1 entries and one L1 entry per page */
-#define IPU6_MMUV2_L2_RANGE (1024 * PAGE_SIZE)
-/* Max L2 blocks per stream */
-#define IPU6_MMUV2_MAX_L2_BLOCKS 2
-/* Max L1 blocks per stream */
-#define IPU6_MMUV2_MAX_L1_BLOCKS 16
-#define IPU6_MMUV2_TRASH_RANGE (IPU6_MMUV2_L2_RANGE * IPU6_MMUV2_MAX_L2_BLOCKS)
-/* Entries per L1 block */
-#define MMUV2_ENTRIES_PER_L1_BLOCK 16
-#define MMUV2_TRASH_L1_BLOCK_OFFSET (MMUV2_ENTRIES_PER_L1_BLOCK * PAGE_SIZE)
-#define MMUV2_TRASH_L2_BLOCK_OFFSET IPU6_MMUV2_L2_RANGE
-
-/*
- * In some of the IPU6 MMUs, there is provision to configure L1 and L2 page
- * table caches. Both these L1 and L2 caches are divided into multiple sections
- * called streams. There is maximum 16 streams for both caches. Each of these
- * sections are subdivided into multiple blocks. When nr_l1streams = 0 and
- * nr_l2streams = 0, means the MMU is of type MMU_V1 and do not support
- * L1/L2 page table caches.
- *
- * L1 stream per block sizes are configurable and varies per usecase.
- * L2 has constant block sizes - 2 blocks per stream.
- *
- * MMU1 support pre-fetching of the pages to have less cache lookup misses. To
- * enable the pre-fetching, MMU1 AT (Address Translator) device registers
- * need to be configured.
- *
- * There are four types of memory accesses which requires ZLW configuration.
- * ZLW(Zero Length Write) is a mechanism to enable VT-d pre-fetching on IOMMU.
- *
- * 1. Sequential Access or 1D mode
- * Set ZLW_EN -> 1
- * set ZLW_PAGE_CROSS_1D -> 1
- * Set ZLW_N to "N" pages so that ZLW will be inserte N pages ahead where
- * N is pre-defined and hardcoded in the platform data
- * Set ZLW_2D -> 0
- *
- * 2. ZLW 2D mode
- * Set ZLW_EN -> 1
- * set ZLW_PAGE_CROSS_1D -> 1,
- * Set ZLW_N -> 0
- * Set ZLW_2D -> 1
- *
- * 3. ZLW Enable (no 1D or 2D mode)
- * Set ZLW_EN -> 1
- * set ZLW_PAGE_CROSS_1D -> 0,
- * Set ZLW_N -> 0
- * Set ZLW_2D -> 0
- *
- * 4. ZLW disable
- * Set ZLW_EN -> 0
- * set ZLW_PAGE_CROSS_1D -> 0,
- * Set ZLW_N -> 0
- * Set ZLW_2D -> 0
- *
- * To configure the ZLW for the above memory access, four registers are
- * available. Hence to track these four settings, we have the following entries
- * in the struct ipu6_mmu_hw. Each of these entries are per stream and
- * available only for the L1 streams.
- *
- * a. l1_zlw_en -> To track zlw enabled per stream (ZLW_EN)
- * b. l1_zlw_1d_mode -> Track 1D mode per stream. ZLW inserted at page boundary
- * c. l1_ins_zlw_ahead_pages -> to track how advance the ZLW need to be
inserted
- * Insert ZLW request N pages ahead address.
- * d. l1_zlw_2d_mode -> To track 2D mode per stream (ZLW_2D)
- *
- *
- * Currently L1/L2 streams, blocks, AT ZLW configurations etc. are pre-defined
- * as per the usecase specific calculations. Any change to this pre-defined
- * table has to happen in sync with IPU6 FW.
- */
-struct ipu6_mmu_hw {
- union {
- unsigned long offset;
- void __iomem *base;
- };
- u32 info_bits;
- u8 nr_l1streams;
- /*
- * L1 has variable blocks per stream - total of 64 blocks and maximum of
- * 16 blocks per stream. Configurable by using the block start address
- * per stream. Block start address is calculated from the block size
- */
- u8 l1_block_sz[IPU6_MMU_MAX_TLB_L1_STREAMS];
- /* Is ZLW is enabled in each stream */
- bool l1_zlw_en[IPU6_MMU_MAX_TLB_L1_STREAMS];
- bool l1_zlw_1d_mode[IPU6_MMU_MAX_TLB_L1_STREAMS];
- u8 l1_ins_zlw_ahead_pages[IPU6_MMU_MAX_TLB_L1_STREAMS];
- bool l1_zlw_2d_mode[IPU6_MMU_MAX_TLB_L1_STREAMS];
-
- u32 l1_stream_id_reg_offset;
- u32 l2_stream_id_reg_offset;
-
- u8 nr_l2streams;
- /*
- * L2 has fixed 2 blocks per stream. Block address is calculated
- * from the block size
- */
- u8 l2_block_sz[IPU6_MMU_MAX_TLB_L2_STREAMS];
- /* flag to track if WA is needed for successive invalidate HW bug */
- bool insert_read_before_invalidate;
-};
-
struct ipu6_isys_csi2_pdata {
void __iomem *base;
};
@@ -274,8 +146,6 @@ struct ipu6_isys_internal_tpg_pdata {
struct ipu6_hw_variants {
unsigned long offset;
- u32 nr_mmus;
- struct ipu6_mmu_hw mmu_hw[IPU6_MMU_MAX_DEVICES];
u8 cdc_fifos;
u8 cdc_fifo_threshold[IPU6_MAX_VC_IOSF_PORTS];
u32 dmem_offset;
_______________________________________________
linuxtv-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
