For devices that need to preserve address offset on mapping through swiotlb, this patch adds offset preserving based on page_offset_mask and keeps the offset if the mask is non zero. This is needed for device drivers like NVMe.
Signed-off-by: Jianxiong Gao <jx...@google.com> --- kernel/dma/swiotlb.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 7c42df6e6100..4cab35f2c9bc 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -468,7 +468,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr, dma_addr_t tbl_dma_addr = phys_to_dma_unencrypted(hwdev, io_tlb_start); unsigned long flags; phys_addr_t tlb_addr; - unsigned int nslots, stride, index, wrap; + unsigned int nslots, stride, index, wrap, page_offset_mask, page_offset; int i; unsigned long mask; unsigned long offset_slots; @@ -500,12 +500,16 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr, ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT); + page_offset_mask = dma_get_page_offset_mask(hwdev); + page_offset = orig_addr & page_offset_mask; + alloc_size += page_offset; + /* * For mappings greater than or equal to a page, we limit the stride * (and hence alignment) to a page size. */ nslots = ALIGN(alloc_size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; - if (alloc_size >= PAGE_SIZE) + if ((alloc_size >= PAGE_SIZE) || (page_offset_mask > (1 << IO_TLB_SHIFT))) stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT)); else stride = 1; @@ -583,6 +587,11 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr, */ for (i = 0; i < nslots; i++) io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT); + /* + * When keeping the offset of the original data, we need to advance + * the tlb_addr by the offset of orig_addr. + */ + tlb_addr += page_offset; if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) swiotlb_bounce(orig_addr, tlb_addr, mapping_size, DMA_TO_DEVICE); @@ -598,7 +607,9 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, enum dma_data_direction dir, unsigned long attrs) { unsigned long flags; - int i, count, nslots = ALIGN(alloc_size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; + unsigned int num_page_offset_slabs, page_offset_mask = dma_get_page_offset_mask(hwdev); + int i, count; + int nslots = ALIGN(alloc_size + tlb_addr & page_offset_mask, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; int index = (tlb_addr - io_tlb_start) >> IO_TLB_SHIFT; phys_addr_t orig_addr = io_tlb_orig_addr[index]; @@ -610,6 +621,14 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) swiotlb_bounce(orig_addr, tlb_addr, mapping_size, DMA_FROM_DEVICE); + /* + * When dma_get_page_offset_mask is used, we may have padded more slabs + * when padding exceeds one slab. We need to move index back to the + * beginning of the padding. + */ + num_page_offset_slabs = (tlb_addr & page_offset_mask) / (1 << IO_TLB_SHIFT); + index -= num_page_offset_slabs; + /* * Return the buffer to the free list by setting the corresponding * entries to indicate the number of contiguous entries available. -- 2.27.0 _______________________________________________ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu