Convert everything not overriding dma-coherence.h to the generic noncoherent ops. The new dma-noncoherent.c file duplicates a lot of the code in dma-default.c, but that file will be gone by the end of this series.
Signed-off-by: Christoph Hellwig <[email protected]> --- arch/mips/Kconfig | 24 +--- arch/mips/include/asm/dma-mapping.h | 2 + arch/mips/loongson32/Kconfig | 2 - arch/mips/mm/Makefile | 1 + arch/mips/mm/dma-noncoherent.c | 208 ++++++++++++++++++++++++++++ arch/mips/pic32/Kconfig | 1 - arch/mips/txx9/Kconfig | 1 - arch/mips/vr41xx/Kconfig | 5 - 8 files changed, 216 insertions(+), 28 deletions(-) create mode 100644 arch/mips/mm/dma-noncoherent.c diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 8bf378651d74..326bd73bc5bf 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -101,7 +101,6 @@ config MIPS_GENERIC select IRQ_MIPS_CPU select LIBFDT select MIPS_CPU_SCACHE - select MIPS_DMA_DEFAULT select MIPS_GIC select MIPS_L1_CACHE_SHIFT_7 select NO_EXCEPT_FILL @@ -145,7 +144,6 @@ config MIPS_ALCHEMY select CEVT_R4K select CSRC_R4K select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select DMA_MAYBE_COHERENT # Au1000,1500,1100 aren't, rest is select SYS_HAS_CPU_MIPS32_R1 select SYS_SUPPORTS_32BIT_KERNEL @@ -161,7 +159,6 @@ config AR7 select CEVT_R4K select CSRC_R4K select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select NO_EXCEPT_FILL select SWAP_IO_SPACE select SYS_HAS_CPU_MIPS32_R1 @@ -204,7 +201,6 @@ config ATH79 select COMMON_CLK select CLKDEV_LOOKUP select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select MIPS_MACHINE select SYS_HAS_CPU_MIPS32_R2 select SYS_HAS_EARLY_PRINTK @@ -262,7 +258,6 @@ config BCM47XX select HW_HAS_PCI select IRQ_MIPS_CPU select SYS_HAS_CPU_MIPS32_R1 - select MIPS_DMA_DEFAULT select NO_EXCEPT_FILL select SYS_SUPPORTS_32BIT_KERNEL select SYS_SUPPORTS_LITTLE_ENDIAN @@ -286,7 +281,6 @@ config BCM63XX select SYNC_R4K select DMA_NONCOHERENT select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select SYS_SUPPORTS_32BIT_KERNEL select SYS_SUPPORTS_BIG_ENDIAN select SYS_HAS_EARLY_PRINTK @@ -309,7 +303,6 @@ config MIPS_COBALT select I8259 select IRQ_MIPS_CPU select IRQ_GT641XX - select MIPS_DMA_DEFAULT select PCI_GT64XXX_PCI0 select PCI select SYS_HAS_CPU_NEVADA @@ -330,7 +323,6 @@ config MACH_DECSTATION select CPU_R4000_WORKAROUNDS if 64BIT select CPU_R4400_WORKAROUNDS if 64BIT select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select NO_IOPORT_MAP select IRQ_MIPS_CPU select SYS_HAS_CPU_R3000 @@ -390,7 +382,6 @@ config MACH_INGENIC select SYS_SUPPORTS_ZBOOT_UART16550 select DMA_NONCOHERENT select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select PINCTRL select GPIOLIB select COMMON_CLK @@ -405,7 +396,6 @@ config LANTIQ select IRQ_MIPS_CPU select CEVT_R4K select CSRC_R4K - select MIPS_DMA_DEFAULT select SYS_HAS_CPU_MIPS32_R1 select SYS_HAS_CPU_MIPS32_R2 select SYS_SUPPORTS_BIG_ENDIAN @@ -433,7 +423,6 @@ config LASAT select SYS_HAS_EARLY_PRINTK select HW_HAS_PCI select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select PCI_GT64XXX_PCI0 select MIPS_NILE4 select R5000_CPU_SCACHE @@ -479,7 +468,6 @@ config MACH_PISTACHIO select LIBFDT select MFD_SYSCON select MIPS_CPU_SCACHE - select MIPS_DMA_DEFAULT select MIPS_GIC select PINCTRL select REGULATOR @@ -512,7 +500,6 @@ config MIPS_MALTA select GENERIC_ISA_DMA select HAVE_PCSPKR_PLATFORM select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select MIPS_GIC select HW_HAS_PCI select I8253 @@ -607,7 +594,6 @@ config PMC_MSP select SYS_SUPPORTS_BIG_ENDIAN select SYS_SUPPORTS_MIPS16 select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select SERIAL_8250 select SERIAL_8250_CONSOLE select USB_EHCI_BIG_ENDIAN_MMIO @@ -625,7 +611,6 @@ config RALINK select BOOT_RAW select DMA_NONCOHERENT select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select USE_OF select SYS_HAS_CPU_MIPS32_R1 select SYS_HAS_CPU_MIPS32_R2 @@ -652,7 +637,6 @@ config SGI_IP22 select I8259 select IP22_CPU_SCACHE select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select GENERIC_ISA_DMA_SUPPORT_BROKEN select SGI_HAS_I8042 select SGI_HAS_INDYDOG @@ -713,7 +697,6 @@ config SGI_IP28 select DMA_NONCOHERENT select GENERIC_ISA_DMA_SUPPORT_BROKEN select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select HW_HAS_EISA select I8253 select I8259 @@ -864,7 +847,6 @@ config SNI_RM select I8253 select I8259 select ISA - select MIPS_DMA_DEFAULT select SWAP_IO_SPACE if CPU_BIG_ENDIAN select SYS_HAS_CPU_R4X00 select SYS_HAS_CPU_R5000 @@ -895,7 +877,6 @@ config MIKROTIK_RB532 select DMA_NONCOHERENT select HW_HAS_PCI select IRQ_MIPS_CPU - select MIPS_DMA_DEFAULT select SYS_HAS_CPU_MIPS32_R1 select SYS_SUPPORTS_32BIT_KERNEL select SYS_SUPPORTS_LITTLE_ENDIAN @@ -1132,7 +1113,12 @@ config DMA_PERDEV_COHERENT config DMA_NONCOHERENT bool + select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select ARCH_HAS_SYNC_DMA_FOR_CPU select NEED_DMA_MAP_STATE + select DMA_NONCOHERENT_MMAP + select DMA_NONCOHERENT_CACHE_SYNC + select DMA_NONCOHERENT_OPS if !MIPS_DMA_DEFAULT config SYS_HAS_EARLY_PRINTK bool diff --git a/arch/mips/include/asm/dma-mapping.h b/arch/mips/include/asm/dma-mapping.h index 7c0d4f0ccaa0..e32a7b439816 100644 --- a/arch/mips/include/asm/dma-mapping.h +++ b/arch/mips/include/asm/dma-mapping.h @@ -19,6 +19,8 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) return &mips_swiotlb_ops; #elif defined(CONFIG_MIPS_DMA_DEFAULT) return &mips_default_dma_map_ops; +#elif defined(CONFIG_DMA_NONCOHERENT_OPS) + return &dma_noncoherent_ops; #else return &dma_direct_ops; #endif diff --git a/arch/mips/loongson32/Kconfig b/arch/mips/loongson32/Kconfig index 7a69a6c0ce22..462b126f45aa 100644 --- a/arch/mips/loongson32/Kconfig +++ b/arch/mips/loongson32/Kconfig @@ -10,7 +10,6 @@ config LOONGSON1_LS1B select CSRC_R4K if !MIPS_EXTERNAL_TIMER select SYS_HAS_CPU_LOONGSON1B select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select BOOT_ELF32 select IRQ_MIPS_CPU select SYS_SUPPORTS_32BIT_KERNEL @@ -27,7 +26,6 @@ config LOONGSON1_LS1C select CSRC_R4K if !MIPS_EXTERNAL_TIMER select SYS_HAS_CPU_LOONGSON1C select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select BOOT_ELF32 select IRQ_MIPS_CPU select SYS_SUPPORTS_32BIT_KERNEL diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile index 038bfed34946..c6146c3805dc 100644 --- a/arch/mips/mm/Makefile +++ b/arch/mips/mm/Makefile @@ -18,6 +18,7 @@ obj-$(CONFIG_64BIT) += pgtable-64.o obj-$(CONFIG_HIGHMEM) += highmem.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_MIPS_DMA_DEFAULT) += dma-default.o +obj-$(CONFIG_DMA_NONCOHERENT) += dma-noncoherent.o obj-$(CONFIG_SWIOTLB) += dma-swiotlb.o obj-$(CONFIG_CPU_R4K_CACHE_TLB) += c-r4k.o cex-gen.o tlb-r4k.o diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c new file mode 100644 index 000000000000..25edf6d6b686 --- /dev/null +++ b/arch/mips/mm/dma-noncoherent.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2000 Ani Joshi <[email protected]> + * Copyright (C) 2000, 2001, 06 Ralf Baechle <[email protected]> + * swiped from i386, and cloned for MIPS by Geert, polished by Ralf. + */ +#include <linux/dma-direct.h> +#include <linux/dma-noncoherent.h> +#include <linux/dma-contiguous.h> +#include <linux/highmem.h> + +#include <asm/cache.h> +#include <asm/cpu-type.h> +#include <asm/dma-coherence.h> +#include <asm/io.h> + +#ifdef CONFIG_DMA_PERDEV_COHERENT +static inline int dev_is_coherent(struct device *dev) +{ + return dev->archdata.dma_coherent; +} +#else +static inline int dev_is_coherent(struct device *dev) +{ + switch (coherentio) { + default: + case IO_COHERENCE_DEFAULT: + return hw_coherentio; + case IO_COHERENCE_ENABLED: + return 1; + case IO_COHERENCE_DISABLED: + return 0; + } +} +#endif /* CONFIG_DMA_PERDEV_COHERENT */ + +/* + * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively + * fill random cachelines with stale data at any time, requiring an extra + * flush post-DMA. + * + * Warning on the terminology - Linux calls an uncached area coherent; MIPS + * terminology calls memory areas with hardware maintained coherency coherent. + * + * Note that the R14000 and R16000 should also be checked for in this condition. + * However this function is only called on non-I/O-coherent systems and only the + * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp. + * SGI IP32 aka O2. + */ +static inline bool cpu_needs_post_dma_flush(struct device *dev) +{ + if (dev_is_coherent(dev)) + return false; + + switch (boot_cpu_type()) { + case CPU_R10000: + case CPU_R12000: + case CPU_BMIPS5000: + return true; + default: + /* + * Presence of MAARs suggests that the CPU supports + * speculatively prefetching data, and therefore requires + * the post-DMA flush/invalidate. + */ + return cpu_has_maar; + } +} + +void *arch_dma_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +{ + void *ret; + + ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + if (!ret) + return NULL; + + if (!dev_is_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) { + dma_cache_wback_inv((unsigned long) ret, size); + ret = UNCAC_ADDR(ret); + } + + return ret; +} + +void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr, unsigned long attrs) +{ + if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_coherent(dev)) + cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr); + dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); +} + +int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + unsigned long user_count = vma_pages(vma); + unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long addr = (unsigned long)cpu_addr; + unsigned long off = vma->vm_pgoff; + unsigned long pfn; + int ret = -ENXIO; + + if (!dev_is_coherent(dev)) + addr = CAC_ADDR(addr); + + pfn = page_to_pfn(virt_to_page((void *)addr)); + + if (attrs & DMA_ATTR_WRITE_COMBINE) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + else + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) + return ret; + + if (off < count && user_count <= (count - off)) { + ret = remap_pfn_range(vma, vma->vm_start, + pfn + off, + user_count << PAGE_SHIFT, + vma->vm_page_prot); + } + + return ret; +} + +static inline void dma_sync_virt(void *addr, size_t size, + enum dma_data_direction dir) +{ + switch (dir) { + case DMA_TO_DEVICE: + dma_cache_wback((unsigned long)addr, size); + break; + + case DMA_FROM_DEVICE: + dma_cache_inv((unsigned long)addr, size); + break; + + case DMA_BIDIRECTIONAL: + dma_cache_wback_inv((unsigned long)addr, size); + break; + + default: + BUG(); + } +} + +/* + * A single sg entry may refer to multiple physically contiguous pages. But + * we still need to process highmem pages individually. If highmem is not + * configured then the bulk of this loop gets optimized out. + */ +static inline void dma_sync_phys(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) +{ + struct page *page = pfn_to_page(paddr >> PAGE_SHIFT); + unsigned long offset = paddr & ~PAGE_MASK; + size_t left = size; + + do { + size_t len = left; + + if (PageHighMem(page)) { + void *addr; + + if (offset + len > PAGE_SIZE) { + if (offset >= PAGE_SIZE) { + page += offset >> PAGE_SHIFT; + offset &= ~PAGE_MASK; + } + len = PAGE_SIZE - offset; + } + + addr = kmap_atomic(page); + dma_sync_virt(addr + offset, len, dir); + kunmap_atomic(addr); + } else + dma_sync_virt(page_address(page) + offset, size, dir); + offset = 0; + page++; + left -= len; + } while (left); +} + +void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir) +{ + if (!dev_is_coherent(dev)) + dma_sync_phys(paddr, size, dir); +} + +void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir) +{ + if (cpu_needs_post_dma_flush(dev)) + dma_sync_phys(paddr, size, dir); +} + +void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, + enum dma_data_direction direction) +{ + BUG_ON(direction == DMA_NONE); + + if (!dev_is_coherent(dev)) + dma_sync_virt(vaddr, size, direction); +} diff --git a/arch/mips/pic32/Kconfig b/arch/mips/pic32/Kconfig index 7feb7359b05b..e284e89183cc 100644 --- a/arch/mips/pic32/Kconfig +++ b/arch/mips/pic32/Kconfig @@ -11,7 +11,6 @@ config PIC32MZDA select CEVT_R4K select CSRC_R4K select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select SYS_HAS_CPU_MIPS32_R2 select SYS_HAS_EARLY_PRINTK select SYS_SUPPORTS_32BIT_KERNEL diff --git a/arch/mips/txx9/Kconfig b/arch/mips/txx9/Kconfig index 9dfda3e90348..d2509c93f0ee 100644 --- a/arch/mips/txx9/Kconfig +++ b/arch/mips/txx9/Kconfig @@ -16,7 +16,6 @@ config MACH_TX49XX config MACH_TXX9 bool select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select SWAP_IO_SPACE select SYS_HAS_EARLY_PRINTK select SYS_SUPPORTS_32BIT_KERNEL diff --git a/arch/mips/vr41xx/Kconfig b/arch/mips/vr41xx/Kconfig index cc69b2f663fa..992c988b83b0 100644 --- a/arch/mips/vr41xx/Kconfig +++ b/arch/mips/vr41xx/Kconfig @@ -9,7 +9,6 @@ config CASIO_E55 select CEVT_R4K select CSRC_R4K select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select IRQ_MIPS_CPU select ISA select SYS_SUPPORTS_32BIT_KERNEL @@ -20,7 +19,6 @@ config IBM_WORKPAD select CEVT_R4K select CSRC_R4K select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select IRQ_MIPS_CPU select ISA select SYS_SUPPORTS_32BIT_KERNEL @@ -31,7 +29,6 @@ config TANBAC_TB022X select CEVT_R4K select CSRC_R4K select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select IRQ_MIPS_CPU select HW_HAS_PCI select SYS_SUPPORTS_32BIT_KERNEL @@ -48,7 +45,6 @@ config VICTOR_MPC30X select CEVT_R4K select CSRC_R4K select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select IRQ_MIPS_CPU select HW_HAS_PCI select PCI_VR41XX @@ -60,7 +56,6 @@ config ZAO_CAPCELLA select CEVT_R4K select CSRC_R4K select DMA_NONCOHERENT - select MIPS_DMA_DEFAULT select IRQ_MIPS_CPU select HW_HAS_PCI select PCI_VR41XX -- 2.17.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
