One feature that was lost from the old `dma_read!` and `dma_write!` when
moving to `io_read!` and `io_write!` was the ability to read/write a large
structs. However, the semantics was unclear to begin with, as there was no
guarantee about their atomicity even for structs that were small enough to
fit in u32. Re-introduce the capability in the form of copying methods.
dma_read!(foo, bar) -> io_project!(foo, bar).copy_read()
dma_write!(foo, bar, baz) -> io_project!(foo, bar).copy_write(baz)
Model these semantics after memcpy so user has clear expectation of lack of
atomicity. As an additional benefit of this change, this now works for MMIO
as well by mapping them to `memcpy_{from,to}io`.
For slices which is DST so the `copy_read` and `copy_write` API above can't
work, add `copy_from_slice` and `copy_to_slice` to copy from/to normal
memory, and `copy_from_io_slice` and `copy_to_io_slice` to copy from/to
other `Io` regions. They're optimized if at least one end is mapped to
system memory; if none are, the copy occurs with an intermediate stack
buffer.
Signed-off-by: Gary Guo <[email protected]>
---
rust/helpers/io.c | 13 ++
rust/kernel/dma.rs | 9 ++
rust/kernel/io.rs | 367 ++++++++++++++++++++++++++++++++++++++++++++++-
samples/rust/rust_dma.rs | 14 +-
4 files changed, 397 insertions(+), 6 deletions(-)
diff --git a/rust/helpers/io.c b/rust/helpers/io.c
index 397810864a24..7ed9a4f77f1b 100644
--- a/rust/helpers/io.c
+++ b/rust/helpers/io.c
@@ -19,6 +19,19 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr)
iounmap(addr);
}
+__rust_helper void rust_helper_memcpy_fromio(void *dst,
+ const volatile void __iomem *src,
+ size_t count)
+{
+ memcpy_fromio(dst, src, count);
+}
+
+__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst,
+ const void *src, size_t count)
+{
+ memcpy_toio(dst, src, count);
+}
+
__rust_helper u8 rust_helper_readb(const void __iomem *addr)
{
return readb(addr);
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
index 68015a2ab43b..71d495a2e3a8 100644
--- a/rust/kernel/dma.rs
+++ b/rust/kernel/dma.rs
@@ -18,6 +18,7 @@
IoBackend,
IoBase,
IoCapable,
+ IoCopyable,
SysMem,
SysMemBackend, //
},
@@ -1196,6 +1197,14 @@ fn io_write<'a>(view: Self::View<'a, T>, value: T) {
}
}
+// SAFETY: `CoherentView::as_ptr` is mapped to CPU address space.
+unsafe impl IoCopyable for CoherentBackend {
+ #[inline(always)]
+ fn is_mapped<T: ?Sized + KnownSize>(_view: Self::View<'_, T>) -> bool {
+ true
+ }
+}
+
impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> {
type Backend = CoherentBackend;
type Target = T;
diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs
index 72f3acc0f50d..2b238b625672 100644
--- a/rust/kernel/io.rs
+++ b/rust/kernel/io.rs
@@ -5,7 +5,8 @@
//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h)
use core::{
- marker::PhantomData, //
+ marker::PhantomData,
+ mem::MaybeUninit, //
};
use crate::{
@@ -229,6 +230,63 @@ pub trait IoCapable<T>: IoBackend {
fn io_write<'a>(view: Self::View<'a, T>, value: T);
}
+/// Trait indicating that an I/O backend supports memory copy operations.
+///
+/// # Safety
+///
+/// If [`Self::is_mapped`] is overridden, it must be correct per documentation.
+pub unsafe trait IoCopyable: IoBackend {
+ /// Whether the pointers for this I/O backend are in the CPU address
space, and are coherently
+ /// mapped.
+ ///
+ /// When this returns true, `Self::as_ptr(view)` must return a valid and
aligned pointer. The
+ /// pointer may be accessed with byte-wise atomic memory copy or volatile
read/write.
+ ///
+ /// This is not an associated constants to support backends where the view
may be conditionally
+ /// mapped. This method should be marked as `#[inline(always)]` if it
always returns true, so
+ /// `build_assert!()` in `copy_{from,to}io` can see it.
+ #[inline]
+ fn is_mapped<T: ?Sized + KnownSize>(_view: Self::View<'_, T>) -> bool {
+ false
+ }
+
+ /// Copy contents of `view` to `buffer`.
+ ///
+ /// # Safety
+ ///
+ /// - `buffer` is valid for volatile write for `view.size()` bytes.
+ #[inline]
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+ build_assert!(Self::is_mapped(view));
+
+ let ptr = Self::as_ptr(view);
+
+ // Use `bindings::memcpy` instead of copy_nonoverlapping for volatile.
+ // SAFETY:
+ // - `is_mapped` guarantees `ptr` is in CPU address space and valid
for read.
+ // - `buffer` is valid for write for `view.size()` bytes which is
equal to `ptr.len()`.
+ unsafe { bindings::memcpy(buffer.cast(), ptr.cast(), ptr.len()) };
+ }
+
+ /// Copy `size` bytes from `buffer` to `address`.
+ ///
+ /// # Safety
+ ///
+ /// - `buffer` is valid for volatile read for `view.size()` bytes.
+ #[inline]
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+ build_assert!(Self::is_mapped(view));
+
+ let ptr = Self::as_ptr(view);
+
+ // Use `bindings::memcpy` instead of copy_nonoverlapping for volatile.
+ // SAFETY:
+ // - `is_mapped` guarantees `ptr` is in CPU address space and valid
for write.
+ // - `buffer` is valid for read for `view.size()` bytes which is equal
to `ptr.len()`.
+ unsafe { bindings::memcpy(ptr.cast(), buffer.cast(), ptr.len()) };
+ }
+}
+
/// Describes a given I/O location: its offset, width, and type to convert the
raw value from and
/// into.
///
@@ -306,6 +364,24 @@ fn size(self) -> usize {
KnownSize::size(Self::Backend::as_ptr(self.as_view()))
}
+ /// Returns the length of the slice in number of elements.
+ #[inline]
+ fn len<T>(self) -> usize
+ where
+ Self: Io<'a, Target = [T]>,
+ {
+ Self::Backend::as_ptr(self.as_view()).len()
+ }
+
+ /// Returns `true` if the slice has a length of 0.
+ #[inline]
+ fn is_empty<T>(self) -> bool
+ where
+ Self: Io<'a, Target = [T]>,
+ {
+ self.len() == 0
+ }
+
/// Try to convert into a different typed I/O view.
///
/// The target type must be of same or smaller size to current type, and
the current view must
@@ -397,6 +473,264 @@ fn write_val(self, value: Self::Target)
Self::Backend::io_write(self.as_view(), value)
}
+ /// Copy-read from I/O memory.
+ ///
+ /// There is no atomicity guarantee.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) {
+ /// // let mmio: Mmio<'_, [u8; 6]>;
+ /// let val: [u8; 6] = mmio.copy_read();
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_read(self) -> Self::Target
+ where
+ Self::Backend: IoCopyable,
+ Self::Target: Sized + FromBytes,
+ {
+ let view = self.as_view();
+
+ // Optimized path if I/O backend is CPU mapped.
+ if Self::Backend::is_mapped(view) {
+ let ptr = Self::Backend::as_ptr(view);
+ // SAFETY:
+ // - `is_mapped` guarantees `ptr` is valid for read in CPU address
space.
+ // - Using read_volatile() here so that race with hardware is
well-defined.
+ // - Using read_volatile() here is not sound if it races with
other CPU per Rust
+ // rules, but this is allowed per LKMM.
+ return unsafe { ptr.read_volatile() };
+ }
+
+ // Project `self` to `[u8]`.
+ let ptr = Self::Backend::as_ptr(view);
+ // SAFETY: This is a identity projection.
+ let slice_view = unsafe {
+ Self::Backend::project_view(
+ view,
+ core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(),
size_of::<Self::Target>()),
+ )
+ };
+
+ let mut buf = MaybeUninit::<Self::Target>::uninit();
+ // SAFETY: `buf.as_mut_ptr()` is valid for write for `size_of::<T>()`
bytes.
+ unsafe { Self::Backend::copy_from_io(slice_view,
buf.as_mut_ptr().cast()) };
+ // SAFETY: T: FromBytes` guarantee that all bit patterns are valid.
+ unsafe { buf.assume_init() }
+ }
+
+ /// Copy-write to I/O memory.
+ ///
+ /// There is no atomicity guarantee.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) {
+ /// // let mmio: Mmio<'_, [u8; 6]>;
+ /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_write(self, value: Self::Target)
+ where
+ Self::Backend: IoCopyable,
+ Self::Target: Sized + AsBytes,
+ {
+ let view = self.as_view();
+
+ // Optimized path if I/O backend is CPU mapped.
+ if Self::Backend::is_mapped(view) {
+ let ptr = Self::Backend::as_ptr(view);
+ // SAFETY:
+ // - `is_mapped` guarantees `ptr` is valid for write in CPU
address space.
+ // - Using write_volatile() here so that race with hardware is
well-defined.
+ // - Using write_volatile() here is not sound if it races with
other CPU per Rust
+ // rules, but this is allowed per LKMM.
+ unsafe { ptr.write_volatile(value) };
+ return;
+ }
+
+ // Project `self` to `[u8]`.
+ let ptr = Self::Backend::as_ptr(view);
+ // SAFETY: This is a identity projection.
+ let slice_view = unsafe {
+ Self::Backend::project_view(
+ view,
+ core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(),
size_of::<Self::Target>()),
+ )
+ };
+
+ // SAFETY: `&raw const value` is valid for read for `size_of::<T>()`
bytes.
+ unsafe { Self::Backend::copy_to_io(slice_view, (&raw const
value).cast()) };
+ core::mem::forget(value);
+ }
+
+ /// Copy bytes from slice to I/O memory.
+ ///
+ /// The length of `self` must be the same as `data`, similar to
[`[u8]::copy_from_slice`].
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
+ /// // let mmio: Mmio<'_, [u8]>;
+ /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_from_slice(self, data: &[u8])
+ where
+ Self::Backend: IoCopyable,
+ Self: Io<'a, Target = [u8]>,
+ {
+ assert_eq!(self.len(), data.len());
+
+ // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes.
+ unsafe {
+ Self::Backend::copy_to_io(self.as_view(), data.as_ptr());
+ }
+ }
+
+ /// Copy bytes from I/O memory to slice.
+ ///
+ /// The length of `self` must be the same as `data`, similar to
[`[u8]::copy_from_slice`].
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
+ /// // let mmio: Mmio<'_, [u8]>;
+ /// let mut buf = [0; 6];
+ /// mmio.copy_to_slice(&mut buf);
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_to_slice(self, data: &mut [u8])
+ where
+ Self::Backend: IoCopyable,
+ Self: Io<'a, Target = [u8]>,
+ {
+ assert_eq!(self.len(), data.len());
+
+ // SAFETY: `data.as_ptr()` is valid for write for `self.size()` bytes.
+ unsafe {
+ Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr());
+ }
+ }
+
+ /// Copy bytes from `data` I/O slice to the `self`.
+ ///
+ /// The length of `self` must be the same as `data`, similar to
[`[u8]::copy_from_slice`].
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_write(dst: Mmio<'_, [u8]>, src: Mmio<'_, [u8]>) {
+ /// // let dst: Mmio<'_, [u8]>;
+ /// // let src: Mmio<'_, [u8]>;
+ /// dst.copy_from_io_slice(src);
+ /// # }
+ /// ```
+ fn copy_from_io_slice<'b, T>(self, data: T)
+ where
+ Self::Backend: IoCopyable,
+ Self: Io<'a, Target = [u8]>,
+ T: Io<'b, Target = [u8], Backend: IoCopyable>,
+ {
+ fn copy_from_io_slice_via_buffer<
+ 'a,
+ 'b,
+ T: Io<'a, Target = [u8], Backend: IoCopyable>,
+ U: Io<'b, Target = [u8], Backend: IoCopyable>,
+ >(
+ dest: T,
+ src: U,
+ ) {
+ let mut buf = MaybeUninit::<[u8; 256]>::uninit();
+
+ let mut offset = 0;
+ let mut len = dest.len();
+
+ while len != 0 {
+ let copy_len = core::cmp::min(len, 256);
+
+ // SAFETY: `buf.as_mut_ptr()` is valid for write for
`copy_len` bytes as `copy_len
+ // <= 256`.
+ unsafe {
+ U::Backend::copy_from_io(
+ io_project!(src, [panic: offset..][panic: ..copy_len]),
+ buf.as_mut_ptr().cast(),
+ )
+ };
+
+ // SAFETY: `buf.as_ptr()` is valid for read for `copy_len`
bytes as `copy_len <=
+ // 256`.
+ unsafe {
+ T::Backend::copy_to_io(
+ io_project!(dest, [panic: offset..][panic:
..copy_len]),
+ buf.as_ptr().cast(),
+ )
+ };
+
+ offset += copy_len;
+ len -= copy_len;
+ }
+ }
+
+ assert_eq!(self.len(), data.len());
+
+ let dst_view = self.as_view();
+ let src_view = data.as_view();
+
+ if T::Backend::is_mapped(src_view) {
+ // SAFETY: `T::Backend::as_ptr(src_view)` is valid for read for
`data.len()`
+ // bytes.
+ unsafe {
+ Self::Backend::copy_to_io(self.as_view(),
T::Backend::as_ptr(src_view).cast())
+ }
+ } else if Self::Backend::is_mapped(dst_view) {
+ // SAFETY: `Self::Backend::as_ptr(dst_view)` is valid for write
for `data.len()`
+ // bytes.
+ unsafe {
+ T::Backend::copy_from_io(data.as_view(),
Self::Backend::as_ptr(dst_view).cast())
+ }
+ } else {
+ copy_from_io_slice_via_buffer(dst_view, src_view)
+ }
+ }
+
+ /// Copy bytes from `self` to the `data` I/O slice.
+ ///
+ /// The length of `self` must be the same as `data`, similar to
[`[u8]::copy_from_slice`].
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_write(dst: Mmio<'_, [u8]>, src: Mmio<'_, [u8]>) {
+ /// // let dst: Mmio<'_, [u8]>;
+ /// // let src: Mmio<'_, [u8]>;
+ /// src.copy_to_io_slice(dst);
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_to_io_slice<'b, T>(self, data: T)
+ where
+ Self::Backend: IoCopyable,
+ Self: Io<'a, Target = [u8]>,
+ T: Io<'b, Target = [u8], Backend: IoCopyable>,
+ {
+ data.copy_from_io_slice(self)
+ }
+
/// Returns a view for a given `offset`, performing compile-time bound
checks.
// Always inline to optimize out error path of `build_assert`.
#[inline(always)]
@@ -987,6 +1321,29 @@ fn io_write(view: <$backend as IoBackend>::View<'_, $ty>,
value: $ty) {
#[cfg(CONFIG_64BIT)]
impl_mmio_io_capable!(MmioBackend, u64, readq, writeq);
+// SAFETY: `is_mapped` is not overridden.
+unsafe impl IoCopyable for MmioBackend {
+ #[inline]
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+ // SAFETY:
+ // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
+ // - `buffer` is valid for write for `view.size()` bytes.
+ unsafe {
+ bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(),
view.size());
+ }
+ }
+
+ #[inline]
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+ // SAFETY:
+ // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
+ // - `buffer` is valid for read for `view.size()` bytes.
+ unsafe {
+ bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size());
+ }
+ }
+}
+
/// [`Mmio`] but using relaxed accessors.
///
/// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO
operands instead of
@@ -1142,6 +1499,14 @@ fn io_write(view: SysMem<'_, $ty>, value: $ty) {
#[cfg(CONFIG_64BIT)]
impl_sysmem_io_capable!(u64);
+// SAFETY: `SysMem::as_ptr` is mapped to the CPU address space.
+unsafe impl IoCopyable for SysMemBackend {
+ #[inline(always)]
+ fn is_mapped<T: ?Sized + KnownSize>(_view: Self::View<'_, T>) -> bool {
+ true
+ }
+}
+
/// System memory region.
///
/// Provides `Io` trait implementation for kernel virtual address ranges,
diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs
index 6727c441658a..0f2597798ff5 100644
--- a/samples/rust/rust_dma.rs
+++ b/samples/rust/rust_dma.rs
@@ -12,7 +12,11 @@
Device,
DmaMask, //
},
- io::io_read,
+ io::{
+ io_project,
+ io_read,
+ Io, //
+ },
page, pci,
prelude::*,
scatterlist::{Owned, SGTable},
@@ -74,11 +78,11 @@ fn probe<'bound>(
// SAFETY: There are no concurrent calls to DMA allocation and
mapping primitives.
unsafe { pdev.dma_set_mask_and_coherent(mask)? };
- let mut ca: CoherentBox<[MyStruct]> =
- CoherentBox::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(),
GFP_KERNEL)?;
+ let ca: Coherent<[MyStruct]> =
+ Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(),
GFP_KERNEL)?;
for (i, value) in TEST_VALUES.into_iter().enumerate() {
- ca.init_at(i, MyStruct::new(value.0, value.1))?;
+ io_project!(ca, [panic: i]).copy_write(MyStruct::new(value.0,
value.1));
}
let size = 4 * page::PAGE_SIZE;
@@ -88,7 +92,7 @@ fn probe<'bound>(
Ok(try_pin_init!(Self {
pdev: pdev.into(),
- ca: ca.into(),
+ ca,
sgt <- sgt,
}))
})
--
2.54.0
