This is an automated email from the ASF dual-hosted git repository. acassis pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nuttx.git
commit 3557791ae21d5006e1c0cca13ce7d0683c73720c Author: Marco Casaroli <[email protected]> AuthorDate: Sun Aug 9 15:08:43 2026 +0200 arch/arm64: Build fork() children from the caller's syscall frame. In a kernel or protected build vfork() is reached through a system call, so the return address and stack pointer that the entry point in arm64_fork_func.S can snapshot for itself belong to the kernel-side stub, not to the caller. A child built from that snapshot resumes at a kernel address on a kernel stack. This is why arm64 selected the fork family only for the flat build. Record what the caller was actually doing instead. arm64_sync_exc passes the exception frame to dispatch_syscall() in x7 -- x0-x6 carry the call number and its six parameters, so x7 is free -- and dispatch_syscall() stores it in xcp.sregs, mirroring what riscv_swint.c does. arm64_fork() then chooses where the caller's registers live: arm64_fork_syscall() when TCB_FLAG_SYSCALL is set, rebuilding the child from xcp.sregs so that it returns from the very same SVC as the parent; arm64_fork_direct() otherwise, which is the flat build and any kernel thread that calls the entry point as a plain function. The stack copy and the relocation of pointers into it are shared by both paths in arm64_fork_stack() and arm64_fork_reloc(). With that in place ARCH_ARM64 can select ARCH_HAVE_VFORK unconditionally. Verified on qemu-armv8a:knsh (BUILD_KERNEL), qemu-armv8a:nsh (BUILD_FLAT) and qemu-armv8a:citest_smp under qemu-system-aarch64: ostest's vfork_test passes on all three, and it was absent from knsh before the change. The protected configurations are build-verified only (fvp-armv8r:pnsh), there being no emulator for them here. Assisted-by: Claude Code:claude-opus-5 Signed-off-by: Marco Casaroli <[email protected]> --- Documentation/guides/fork_vfork_migration.rst | 7 +- arch/Kconfig | 2 +- arch/arm64/include/irq.h | 15 + arch/arm64/src/common/arm64_fork.c | 400 ++++++++++++++++++++------ arch/arm64/src/common/arm64_syscall.c | 6 +- arch/arm64/src/common/arm64_vectors.S | 7 + 6 files changed, 341 insertions(+), 96 deletions(-) diff --git a/Documentation/guides/fork_vfork_migration.rst b/Documentation/guides/fork_vfork_migration.rst index 8420fbe5b7c..63e5f682a03 100644 --- a/Documentation/guides/fork_vfork_migration.rst +++ b/Documentation/guides/fork_vfork_migration.rst @@ -153,10 +153,15 @@ exception frame when it traps -- ``xcp.sregs`` is the field that exists for this -- and build the child from that instead, while a kernel thread that calls the entry point directly still takes the ordinary path. -Two architectures do it, and they are worth copying: +Three architectures do it, and they are worth copying: * RISC-V: ``riscv_swint.c`` stores the frame in ``xcp.sregs``, and ``riscv_fork.c`` rebuilds the child from it. +* arm64: ``arm64_vectors.S`` hands the frame to ``dispatch_syscall()``, which + stores it in ``xcp.sregs``; ``arm64_fork()`` then dispatches to + ``arm64_fork_syscall()`` or ``arm64_fork_direct()`` according to whether + ``TCB_FLAG_SYSCALL`` is set, so a kernel thread that calls the entry point + directly still works. * armv7-a: ``arm_syscall.c`` stores the frame in ``xcp.sregs``, and ``arm_fork()`` dispatches to ``arm_fork_syscall()`` or ``arm_fork_direct()``. The discriminator here is a saved user stack diff --git a/arch/Kconfig b/arch/Kconfig index 3a09b26fcfc..3944e83d5a5 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -29,7 +29,7 @@ config ARCH_ARM64 select ARCH_64BIT select ARCH_HAVE_BACKTRACE select ARCH_HAVE_INTERRUPTSTACK - select ARCH_HAVE_VFORK if !BUILD_KERNEL && !BUILD_PROTECTED + select ARCH_HAVE_VFORK select ARCH_HAVE_STACKCHECK select ARCH_HAVE_CUSTOMOPT select ARCH_HAVE_STDARG_H diff --git a/arch/arm64/include/irq.h b/arch/arm64/include/irq.h index 3fae95b1ff3..6dd9a4e2821 100644 --- a/arch/arm64/include/irq.h +++ b/arch/arm64/include/irq.h @@ -284,6 +284,21 @@ struct xcptcontext uint64_t *initregs; #endif +#ifdef CONFIG_LIB_SYSCALL + /* The caller's register context, as saved by the SVC exception entry, for + * the duration of a system call. This is what the *user* was doing when + * it trapped, as opposed to `regs' above, which during a system call + * describes the kernel. + * + * vfork() and fork() need it: they are reached through a system call, so + * the return address and stack pointer their architecture entry point can + * see for itself are the kernel's, and a child built from those would + * resume at a kernel address on a kernel stack. + */ + + uint64_t *sregs; +#endif + #ifdef CONFIG_ARCH_FPU uint64_t *fpu_regs; uint64_t *saved_fpu_regs; diff --git a/arch/arm64/src/common/arm64_fork.c b/arch/arm64/src/common/arm64_fork.c index b7d5af1f9f7..3566c64ded8 100644 --- a/arch/arm64/src/common/arm64_fork.c +++ b/arch/arm64/src/common/arm64_fork.c @@ -53,74 +53,131 @@ ****************************************************************************/ /**************************************************************************** - * Public Functions + * Private Functions ****************************************************************************/ -#ifdef CONFIG_ARCH_FPU +/**************************************************************************** + * Name: arm64_fork_stack + * + * Description: + * Give the child the part of the parent's stack that is in use, copied to + * the top of the child's own stack. + * + * The copy is aligned with the top of each stack rather than the bottom, + * so a single offset carries any address in the copied region from the + * parent's stack to the child's; that offset is what is returned, and + * arm64_fork_reloc() applies it. + * + * Input Parameters: + * parent - The parent task's TCB + * child - The child task's TCB + * sp - The parent's stack pointer where the primitive was called + * + * Returned Value: + * The offset from an address in the parent's stack to the same place in + * the child's copy of it. + * + ****************************************************************************/ -void arm64_fork_fpureg_save(struct fork_s *context) +static uint64_t arm64_fork_stack(struct tcb_s *parent, struct tcb_s *child, + uint64_t sp) { - /* Take a snapshot of the thread fpu reg context right now */ + uint64_t stacktop; + uint64_t stackutil; + uint64_t newtop; - arm64_fpu_save(context->fpu); - UP_DSB(); -} + /* How much of the parent's stack was utilized? The ARM uses a push-down + * stack so that the current stack pointer should be lower than the + * initial, adjusted stack pointer. The stack usage should be the + * difference between those two. + */ -#endif + stacktop = (uint64_t)parent->stack_base_ptr + parent->adj_stack_size; + DEBUGASSERT(stacktop > sp); + stackutil = stacktop - sp; + + /* Make some feeble effort to preserve the stack contents. This is + * feeble because the stack surely contains invalid pointers and other + * content that will not work in the child context. However, if the + * user follows all of the caveats of vfork() usage, even this feeble + * effort is overkill. + */ + + newtop = (uint64_t)child->stack_base_ptr + child->adj_stack_size; + memcpy((void *)(newtop - stackutil), (const void *)sp, stackutil); + + return newtop - stacktop; +} /**************************************************************************** - * Name: arm64_fork + * Name: arm64_fork_reloc * * Description: - * The common ARM64 worker behind up_fork(). vfork() and fork() snapshot - * the caller's registers identically; `vfork' says which primitive was - * called, and is passed straight through to nxtask_setup_fork(), which is - * where the memory semantics are decided. + * Carry one address from the parent's stack over to the child's copy of + * it. Addresses outside the region that arm64_fork_stack() copied are + * returned unchanged: they point somewhere the child shares with the + * parent, or somewhere that has no counterpart at all. * - * The overall sequence is: + * Input Parameters: + * parent - The parent task's TCB + * addr - The address to relocate + * sp - The parent's stack pointer where the primitive was called, + * which is the low end of the region that was copied + * offset - The offset returned by arm64_fork_stack() * - * 1) User code calls vfork() or fork(). up_fork() collects context - * information and transfers control to arm64_fork(). - * 2) arm64_fork() and calls nxtask_setup_fork(). - * 3) nxtask_setup_fork() allocates and configures the child task's TCB. - * This consists of: - * - Allocation of the child task's TCB. - * - Initialization of file descriptors and streams - * - Configuration of environment variables - * - Allocate and initialize the stack - * - Setup the input parameters for the task. - * - Initialization of the TCB (including call to up_initial_state()) - * 4) arm64_fork() provides any additional operating context. arm64_fork - * must: - * - Initialize special values in any CPU registers that were not - * already configured by up_initial_state() - * 5) arm64_fork() then calls nxtask_start_fork() - * 6) nxtask_start_fork() then executes the child thread. + * Returned Value: + * The relocated address. * - * nxtask_abort_fork() may be called if an error occurs between steps 3 and - * 6. + ****************************************************************************/ + +static uint64_t arm64_fork_reloc(struct tcb_s *parent, uint64_t addr, + uint64_t sp, uint64_t offset) +{ + uint64_t stacktop = (uint64_t)parent->stack_base_ptr + + parent->adj_stack_size; + + /* The top of the stack is included: a stack pointer resting there is one + * past the last byte copied, and still has to move with it. + */ + + if (addr >= sp && addr <= stacktop) + { + return addr + offset; + } + + return addr; +} + +/**************************************************************************** + * Name: arm64_fork_direct + * + * Description: + * Clone a caller that reached up_fork() by an ordinary function call, so + * that the register snapshot taken by arm64_fork_func.S describes the + * caller itself. That is the case in a flat build, and for a kernel + * thread in any build. + * + * The child has no exception frame to inherit, so one is synthesised: it + * resumes at the caller's return address, at the same privilege level, + * with the callee-saved registers the caller had. * * Input Parameters: * vfork - true for vfork(), false for fork() - * context - Caller context information saved by up_fork() + * parent - The calling task's TCB + * context - Caller context information saved by arm64_fork_func.S * * Returned Value: - * Upon successful completion, fork() returns 0 to the child process and - * returns the process ID of the child process to the parent process. - * Otherwise, -1 is returned to the parent, no child process is created, - * and errno is set to indicate the error. + * The process ID of the child, or ERROR on failure. * ****************************************************************************/ -pid_t arm64_fork(bool vfork, const struct fork_s *context) +static pid_t arm64_fork_direct(bool vfork, struct tcb_s *parent, + const struct fork_s *context) { - struct tcb_s *parent = this_task(); struct tcb_s *child; + uint64_t offset; uint64_t newsp; uint64_t newfp; - uint64_t newtop; - uint64_t stacktop; - uint64_t stackutil; /* Allocate and initialize a TCB for the child task. */ @@ -131,57 +188,11 @@ pid_t arm64_fork(bool vfork, const struct fork_s *context) return (pid_t)ERROR; } - /* How much of the parent's stack was utilized? The ARM uses - * a push-down stack so that the current stack pointer should - * be lower than the initial, adjusted stack pointer. The - * stack usage should be the difference between those two. - */ - - stacktop = (uint64_t)parent->stack_base_ptr + - parent->adj_stack_size; - DEBUGASSERT(stacktop > context->sp); - stackutil = stacktop - context->sp; + /* Copy the parent's stack to the child and relocate the pointers into it */ - if (child->stack_base_ptr == parent->stack_base_ptr) - { - /* The child is running at the parent's stack addresses, inside its - * own duplicated address environment. There is nothing to relocate: - * every stack address the child inherits is still the address it - * names. - */ - - newsp = context->sp; - newfp = context->fp; - } - else - { - /* Make some feeble effort to preserve the stack contents. This is - * feeble because the stack surely contains invalid pointers and other - * content that will not work in the child context. However, if the - * user follows all of the caveats of vfork() usage, even this feeble - * effort is overkill. - * - * For a POSIX fork() child the stack contents are not merely a feeble - * effort: the child is entitled to use them, and it does. - */ - - newtop = (uint64_t)child->stack_base_ptr + - child->adj_stack_size; - newsp = newtop - stackutil; - memcpy((void *)newsp, (const void *)context->sp, stackutil); - - /* Was there a frame pointer in place before? */ - - if (context->fp >= context->sp && context->fp < stacktop) - { - uint64_t frameutil = stacktop - context->fp; - newfp = newtop - frameutil; - } - else - { - newfp = context->fp; - } - } + offset = arm64_fork_stack(parent, child, context->sp); + newsp = context->sp + offset; + newfp = arm64_fork_reloc(parent, context->fp, context->sp, offset); /* Update the stack pointer, frame pointer, and volatile registers. When * the child TCB was initialized, all of the values were set to zero. @@ -253,3 +264,206 @@ pid_t arm64_fork(bool vfork, const struct fork_s *context) return nxtask_start_fork(child, vfork); } + +#ifdef CONFIG_LIB_SYSCALL + +/**************************************************************************** + * Name: arm64_fork_syscall + * + * Description: + * Clone a caller that reached up_fork() through a system call. The + * register snapshot taken by arm64_fork_func.S is useless here: it + * describes the kernel-side stub, so a child built from it would resume at + * a kernel address on a kernel stack. What the caller was actually doing + * is the exception frame the SVC handler recorded in xcp.sregs; the child + * is built from that. + * + * The child therefore returns from the very same SVC instruction as the + * parent, at the same privilege level, differing only in that it sees 0 + * as the return value and runs on its own stack. + * + * Input Parameters: + * vfork - true for vfork(), false for fork() + * parent - The calling task's TCB + * + * Returned Value: + * The process ID of the child, or ERROR on failure. + * + ****************************************************************************/ + +static pid_t arm64_fork_syscall(bool vfork, struct tcb_s *parent) +{ + uint64_t *sregs = parent->xcp.sregs; + struct tcb_s *child; + uint64_t offset; + uint64_t newsp; + uint64_t newfp; + uint64_t regtop; + uint64_t sp; + + DEBUGASSERT(sregs != NULL); + + /* Which stack the caller was on depends on the level it trapped from: a + * user task uses SP_EL0, anything running in the kernel uses SP_ELx. + */ + + if ((sregs[REG_SPSR] & SPSR_MODE_MASK) == SPSR_MODE_EL0T) + { + sp = sregs[REG_SP_EL0]; + } + else + { + sp = sregs[REG_SP_ELX]; + } + + /* Allocate and initialize a TCB for the child task. The child resumes at + * the instruction after the SVC, which is where the parent resumes too. + */ + + child = nxtask_setup_fork((start_t)sregs[REG_ELR], vfork); + if (!child) + { + serr("ERROR: nxtask_setup_fork failed\n"); + return (pid_t)ERROR; + } + + /* Copy the parent's stack to the child and relocate the pointers into it */ + + offset = arm64_fork_stack(parent, child, sp); + newsp = sp + offset; + newfp = arm64_fork_reloc(parent, sregs[REG_FP], sp, offset); + + /* Where does the register save area the child is resumed from go? The + * parent's is wherever SP_ELx pointed when it trapped, so put the child's + * at the matching place: the top of its own kernel stack if it has one -- + * a user process in a kernel build -- or else the same offset into its + * copy of the parent's stack. + */ + +#ifdef CONFIG_ARCH_KERNEL_STACK + if (child->xcp.kstack) + { + regtop = (uint64_t)child->xcp.kstack + ARCH_KERNEL_STACKSIZE; + } + else +#endif + { + regtop = arm64_fork_reloc(parent, sregs[REG_SP_ELX], sp, offset); + } + + child->xcp.regs = (void *)(regtop - XCPTCONTEXT_SIZE); + + /* Inherit the parent's whole exception frame, integer and FPU registers + * alike, then fix up only what has to differ. + */ + + memcpy(child->xcp.regs, sregs, XCPTCONTEXT_SIZE); + +#ifdef CONFIG_ARCH_FPU + child->xcp.fpu_regs = (void *)(regtop - FPU_CONTEXT_SIZE); +#endif + + child->xcp.regs[REG_X0] = 0; + child->xcp.regs[REG_FP] = newfp; + child->xcp.regs[REG_EXE_DEPTH] = 0; + child->xcp.regs[REG_SP_ELX] = regtop - XCPTCONTEXT_SIZE; + + if ((sregs[REG_SPSR] & SPSR_MODE_MASK) == SPSR_MODE_EL0T) + { + child->xcp.regs[REG_SP_EL0] = newsp; +#ifdef CONFIG_ARCH_KERNEL_STACK + child->xcp.ustkptr = (uintptr_t *)newsp; +#endif + } + + /* And, finally, start the child task. On a failure, nxtask_start_fork() + * will discard the TCB by calling nxtask_abort_fork(). + */ + + return nxtask_start_fork(child, vfork); +} + +#endif /* CONFIG_LIB_SYSCALL */ + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +#ifdef CONFIG_ARCH_FPU + +void arm64_fork_fpureg_save(struct fork_s *context) +{ + /* Take a snapshot of the thread fpu reg context right now */ + + arm64_fpu_save(context->fpu); + UP_DSB(); +} + +#endif + +/**************************************************************************** + * Name: arm64_fork + * + * Description: + * The common ARM64 worker behind up_fork(). vfork() and fork() snapshot + * the caller's registers identically; `vfork' says which primitive was + * called, and is passed straight through to nxtask_setup_fork(), which is + * where the memory semantics are decided. + * + * The overall sequence is: + * + * 1) User code calls vfork() or fork(). up_fork() collects context + * information and transfers control to arm64_fork(). + * 2) arm64_fork() and calls nxtask_setup_fork(). + * 3) nxtask_setup_fork() allocates and configures the child task's TCB. + * This consists of: + * - Allocation of the child task's TCB. + * - Initialization of file descriptors and streams + * - Configuration of environment variables + * - Allocate and initialize the stack + * - Setup the input parameters for the task. + * - Initialization of the TCB (including call to up_initial_state()) + * 4) arm64_fork() provides any additional operating context. arm64_fork + * must: + * - Initialize special values in any CPU registers that were not + * already configured by up_initial_state() + * 5) arm64_fork() then calls nxtask_start_fork() + * 6) nxtask_start_fork() then executes the child thread. + * + * nxtask_abort_fork() may be called if an error occurs between steps 3 and + * 6. + * + * Everything above is common to the two ways this can be reached, which + * differ only in where the caller's registers are to be found -- see + * arm64_fork_direct() and arm64_fork_syscall(). + * + * Input Parameters: + * vfork - true for vfork(), false for fork() + * context - Caller context information saved by up_fork() + * + * Returned Value: + * Upon successful completion, fork() returns 0 to the child process and + * returns the process ID of the child process to the parent process. + * Otherwise, -1 is returned to the parent, no child process is created, + * and errno is set to indicate the error. + * + ****************************************************************************/ + +pid_t arm64_fork(bool vfork, const struct fork_s *context) +{ + struct tcb_s *parent = this_task(); + +#ifdef CONFIG_LIB_SYSCALL + /* If a system call is in progress then this was reached from its kernel- + * side stub, and the caller is the user task that trapped, not the code + * that called into arm64_fork_func.S. + */ + + if ((parent->flags & TCB_FLAG_SYSCALL) != 0) + { + return arm64_fork_syscall(vfork, parent); + } +#endif + + return arm64_fork_direct(vfork, parent, context); +} diff --git a/arch/arm64/src/common/arm64_syscall.c b/arch/arm64/src/common/arm64_syscall.c index 8d503aa64e3..8e9bc569986 100644 --- a/arch/arm64/src/common/arm64_syscall.c +++ b/arch/arm64/src/common/arm64_syscall.c @@ -95,7 +95,7 @@ static void arm64_dump_syscall(const char *tag, uint64_t cmd, uintptr_t dispatch_syscall(unsigned int nbr, uintptr_t parm1, uintptr_t parm2, uintptr_t parm3, uintptr_t parm4, uintptr_t parm5, - uintptr_t parm6) + uintptr_t parm6, void *context) { struct tcb_s *rtcb = this_task(); register long x0 asm("x0") = (long)(nbr); @@ -117,6 +117,10 @@ uintptr_t dispatch_syscall(unsigned int nbr, uintptr_t parm1, return -ENOSYS; } + /* Set the user register context to TCB */ + + rtcb->xcp.sregs = context; + /* Indicate that we are in a syscall handler */ rtcb->flags |= TCB_FLAG_SYSCALL; diff --git a/arch/arm64/src/common/arm64_vectors.S b/arch/arm64/src/common/arm64_vectors.S index eaedd7508be..2dae675644e 100644 --- a/arch/arm64/src/common/arm64_vectors.S +++ b/arch/arm64/src/common/arm64_vectors.S @@ -172,6 +172,13 @@ SECTION_FUNC(text, arm64_sync_exc) msr daifclr, #IRQ_DAIF_MASK /* Re-enable interrupts */ 1: + /* Pass the caller's exception frame as the last argument. x0-x6 hold + * the system call number and its six parameters, so x7 is free. The + * cloning primitives need this frame to build the child's context; see + * dispatch_syscall() and arm64_fork(). + */ + + mov x7, sp bl dispatch_syscall msr daifset, #IRQ_DAIF_MASK /* Disable interrupts */
