The branch main has been updated by brooks: URL: https://cgit.FreeBSD.org/src/commit/?id=fc9b28879e8935cc830b83d8a36d85b9b98b276a
commit fc9b28879e8935cc830b83d8a36d85b9b98b276a Author: Brooks Davis <[email protected]> AuthorDate: 2026-06-21 15:47:53 +0000 Commit: Brooks Davis <[email protected]> CommitDate: 2026-06-21 16:13:27 +0000 uiomove_*: centralize the copy function selection Add a uiomove_step() for the central set of switch statements which choose between userspace and kernel and if data is going to or from the iovec. Refactor uiomove_fromphys loops to unconditionally free per-iteration resources and drop gotos. While here, switch from bcopy to memcpy. Reviewed by: kib Suggested by: emaste Sponsored by: Innovate UK Differential Revision: https://reviews.freebsd.org/D57688 --- sys/amd64/amd64/uio_machdep.c | 34 ++------------------ sys/arm/arm/uio_machdep.c | 33 ++------------------ sys/arm64/arm64/uio_machdep.c | 36 ++-------------------- sys/i386/i386/uio_machdep.c | 33 ++------------------ sys/kern/subr_uio.c | 65 ++++++++++++++++++++++----------------- sys/powerpc/powerpc/uio_machdep.c | 33 ++------------------ sys/riscv/riscv/uio_machdep.c | 36 ++-------------------- sys/sys/uio.h | 1 + 8 files changed, 56 insertions(+), 215 deletions(-) diff --git a/sys/amd64/amd64/uio_machdep.c b/sys/amd64/amd64/uio_machdep.c index 11e6ad2b1da9..9fdfc898172b 100644 --- a/sys/amd64/amd64/uio_machdep.c +++ b/sys/amd64/amd64/uio_machdep.c @@ -96,38 +96,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) &ma[offset >> PAGE_SHIFT], &vaddr, 1, true); cp = (char *)vaddr + page_offset; } - switch (uio->uio_segflg) { - case UIO_USERSPACE: - maybe_yield(); - switch (uio->uio_rw) { - case UIO_READ: - error = copyout(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - error = copyin(iov->iov_base, cp, cnt); - break; - } - if (error) - goto out; - break; - case UIO_SYSSPACE: - switch (uio->uio_rw) { - case UIO_READ: - bcopy(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - bcopy(iov->iov_base, cp, cnt); - break; - } - break; - case UIO_NOCOPY: - break; - } + error = uiomove_step(cp, iov->iov_base, cnt, uio); if (__predict_false(mapped)) { pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1, true); mapped = false; } + if (error != 0) + break; iov->iov_base = (char *)iov->iov_base + cnt; iov->iov_len -= cnt; uio->uio_resid -= cnt; @@ -135,10 +111,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) offset += cnt; n -= cnt; } -out: - if (__predict_false(mapped)) - pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1, - true); if (save == 0) td->td_pflags &= ~TDP_DEADLKTREAT; return (error); diff --git a/sys/arm/arm/uio_machdep.c b/sys/arm/arm/uio_machdep.c index 6b8e4352e06f..7df6104b25fc 100644 --- a/sys/arm/arm/uio_machdep.c +++ b/sys/arm/arm/uio_machdep.c @@ -91,36 +91,10 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) cnt = min(cnt, PAGE_SIZE - page_offset); sf = sf_buf_alloc(ma[offset >> PAGE_SHIFT], 0); cp = (char*)sf_buf_kva(sf) + page_offset; - switch (uio->uio_segflg) { - case UIO_USERSPACE: - maybe_yield(); - switch (uio->uio_rw) { - case UIO_READ: - error = copyout(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - error = copyin(iov->iov_base, cp, cnt); - break; - } - if (error) { - sf_buf_free(sf); - goto out; - } - break; - case UIO_SYSSPACE: - switch (uio->uio_rw) { - case UIO_READ: - bcopy(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - bcopy(iov->iov_base, cp, cnt); - break; - } - break; - case UIO_NOCOPY: - break; - } + error = uiomove_step(cp, iov->iov_base, cnt, uio); sf_buf_free(sf); + if (error != 0) + break; iov->iov_base = (char *)iov->iov_base + cnt; iov->iov_len -= cnt; uio->uio_resid -= cnt; @@ -128,7 +102,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) offset += cnt; n -= cnt; } -out: if (save == 0) td->td_pflags &= ~TDP_DEADLKTREAT; return (error); diff --git a/sys/arm64/arm64/uio_machdep.c b/sys/arm64/arm64/uio_machdep.c index 976055a69491..77f596498c45 100644 --- a/sys/arm64/arm64/uio_machdep.c +++ b/sys/arm64/arm64/uio_machdep.c @@ -92,38 +92,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) &ma[offset >> PAGE_SHIFT], &vaddr, 1, true); cp = (char *)vaddr + page_offset; } - switch (uio->uio_segflg) { - case UIO_USERSPACE: - maybe_yield(); - switch (uio->uio_rw) { - case UIO_READ: - error = copyout(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - error = copyin(iov->iov_base, cp, cnt); - break; - } - if (error) - goto out; - break; - case UIO_SYSSPACE: - switch (uio->uio_rw) { - case UIO_READ: - bcopy(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - bcopy(iov->iov_base, cp, cnt); - break; - } - break; - case UIO_NOCOPY: - break; - } + error = uiomove_step(cp, iov->iov_base, cnt, uio); if (__predict_false(mapped)) { pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1, true); mapped = false; } + if (error != 0) + break; iov->iov_base = (char *)iov->iov_base + cnt; iov->iov_len -= cnt; uio->uio_resid -= cnt; @@ -131,12 +107,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) offset += cnt; n -= cnt; } -out: - if (__predict_false(mapped)) { - panic("ARM64TODO: uiomove_fromphys"); - pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1, - true); - } if (save == 0) td->td_pflags &= ~TDP_DEADLKTREAT; return (error); diff --git a/sys/i386/i386/uio_machdep.c b/sys/i386/i386/uio_machdep.c index b41460aef5b4..674de877d2a1 100644 --- a/sys/i386/i386/uio_machdep.c +++ b/sys/i386/i386/uio_machdep.c @@ -91,38 +91,11 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) sched_pin(); sf = sf_buf_alloc(ma[offset >> PAGE_SHIFT], SFB_CPUPRIVATE); cp = (char *)sf_buf_kva(sf) + page_offset; - switch (uio->uio_segflg) { - case UIO_USERSPACE: - maybe_yield(); - switch (uio->uio_rw) { - case UIO_READ: - error = copyout(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - error = copyin(iov->iov_base, cp, cnt); - break; - } - if (error) { - sf_buf_free(sf); - sched_unpin(); - goto out; - } - break; - case UIO_SYSSPACE: - switch (uio->uio_rw) { - case UIO_READ: - bcopy(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - bcopy(iov->iov_base, cp, cnt); - break; - } - break; - case UIO_NOCOPY: - break; - } + error = uiomove_step(cp, iov->iov_base, cnt, uio); sf_buf_free(sf); sched_unpin(); + if (error != 0) + break; iov->iov_base = (char *)iov->iov_base + cnt; iov->iov_len -= cnt; uio->uio_resid -= cnt; diff --git a/sys/kern/subr_uio.c b/sys/kern/subr_uio.c index fa5865fbe938..fea1395a7f77 100644 --- a/sys/kern/subr_uio.c +++ b/sys/kern/subr_uio.c @@ -200,6 +200,40 @@ uiomove_nofault(void *cp, int n, struct uio *uio) return (uiomove_faultflag(cp, n, uio, 1)); } +int +uiomove_step(void *cp, void *base, size_t len, struct uio *uio) +{ + int error = 0; + + switch (uio->uio_segflg) { + case UIO_USERSPACE: + maybe_yield(); + switch (uio->uio_rw) { + case UIO_READ: + error = copyout(cp, base, len); + break; + case UIO_WRITE: + error = copyin(base, cp, len); + break; + } + break; + case UIO_SYSSPACE: + switch (uio->uio_rw) { + case UIO_READ: + memcpy(base, cp, len); + break; + case UIO_WRITE: + memcpy(cp, base, len); + break; + } + break; + case UIO_NOCOPY: + break; + } + + return (error); +} + static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault) { @@ -246,34 +280,9 @@ uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault) if (cnt > n) cnt = n; - switch (uio->uio_segflg) { - case UIO_USERSPACE: - maybe_yield(); - switch (uio->uio_rw) { - case UIO_READ: - error = copyout(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - error = copyin(iov->iov_base, cp, cnt); - break; - } - if (error) - goto out; - break; - - case UIO_SYSSPACE: - switch (uio->uio_rw) { - case UIO_READ: - bcopy(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - bcopy(iov->iov_base, cp, cnt); - break; - } - break; - case UIO_NOCOPY: - break; - } + error = uiomove_step(cp, iov->iov_base, cnt, uio); + if (error != 0) + goto out; iov->iov_base = (char *)iov->iov_base + cnt; iov->iov_len -= cnt; uio->uio_resid -= cnt; diff --git a/sys/powerpc/powerpc/uio_machdep.c b/sys/powerpc/powerpc/uio_machdep.c index 63911963279a..05d06a95f544 100644 --- a/sys/powerpc/powerpc/uio_machdep.c +++ b/sys/powerpc/powerpc/uio_machdep.c @@ -97,36 +97,10 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) sf = sf_buf_alloc(m, 0); cp = (char*)sf_buf_kva(sf) + page_offset; - switch (uio->uio_segflg) { - case UIO_USERSPACE: - maybe_yield(); - switch (uio->uio_rw) { - case UIO_READ: - error = copyout(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - error = copyin(iov->iov_base, cp, cnt); - break; - } - if (error) { - sf_buf_free(sf); - goto out; - } - break; - case UIO_SYSSPACE: - switch (uio->uio_rw) { - case UIO_READ: - bcopy(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - bcopy(iov->iov_base, cp, cnt); - break; - } - break; - case UIO_NOCOPY: - break; - } + error = uiomove_step(cp, iov->iov_base, cnt, uio); sf_buf_free(sf); + if (error != 0) + break; iov->iov_base = (char *)iov->iov_base + cnt; iov->iov_len -= cnt; uio->uio_resid -= cnt; @@ -134,7 +108,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) offset += cnt; n -= cnt; } -out: if (save == 0) td->td_pflags &= ~TDP_DEADLKTREAT; return (error); diff --git a/sys/riscv/riscv/uio_machdep.c b/sys/riscv/riscv/uio_machdep.c index f171feb1a4bd..77f596498c45 100644 --- a/sys/riscv/riscv/uio_machdep.c +++ b/sys/riscv/riscv/uio_machdep.c @@ -92,38 +92,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) &ma[offset >> PAGE_SHIFT], &vaddr, 1, true); cp = (char *)vaddr + page_offset; } - switch (uio->uio_segflg) { - case UIO_USERSPACE: - maybe_yield(); - switch (uio->uio_rw) { - case UIO_READ: - error = copyout(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - error = copyin(iov->iov_base, cp, cnt); - break; - } - if (error) - goto out; - break; - case UIO_SYSSPACE: - switch (uio->uio_rw) { - case UIO_READ: - bcopy(cp, iov->iov_base, cnt); - break; - case UIO_WRITE: - bcopy(iov->iov_base, cp, cnt); - break; - } - break; - case UIO_NOCOPY: - break; - } + error = uiomove_step(cp, iov->iov_base, cnt, uio); if (__predict_false(mapped)) { pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1, true); mapped = false; } + if (error != 0) + break; iov->iov_base = (char *)iov->iov_base + cnt; iov->iov_len -= cnt; uio->uio_resid -= cnt; @@ -131,12 +107,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) offset += cnt; n -= cnt; } -out: - if (__predict_false(mapped)) { - panic("TODO 3"); - pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1, - true); - } if (save == 0) td->td_pflags &= ~TDP_DEADLKTREAT; return (error); diff --git a/sys/sys/uio.h b/sys/sys/uio.h index 58bb7b13b253..217f65365cb0 100644 --- a/sys/sys/uio.h +++ b/sys/sys/uio.h @@ -98,6 +98,7 @@ int uiomove_fromphys(struct vm_page *ma[], vm_offset_t offset, int n, struct uio *uio); int uiomove_nofault(void *cp, int n, struct uio *uio); int uiomove_object(struct vm_object *obj, off_t obj_size, struct uio *uio); +int uiomove_step(void *cp, void *base, size_t cnt, struct uio *uio); #else /* !_KERNEL */
