The branch main has been updated by br: URL: https://cgit.FreeBSD.org/src/commit/?id=d7a393095cfd51d473a7c6b5e369e69a172f0c37
commit d7a393095cfd51d473a7c6b5e369e69a172f0c37 Author: Ruslan Bukin <[email protected]> AuthorDate: 2026-06-24 07:48:58 +0000 Commit: Ruslan Bukin <[email protected]> CommitDate: 2026-06-24 08:03:46 +0000 riscv: Vector Extension (RVV) support. RVV is a scalable SIMD (Single Instruction, Multiple Data) extension designed to accelerate data-intensive workload such as AI, machine-learning and DSP. RVV exposes vector-length agnostic (VLA) execution and programming model, with implementation defined vector register file size, dynamic vector length selection, flexible register grouping, and rich instruction semantics, serving as the foundation for portable, high-throughput data-parallel acceleration. Spec: https://github.com/riscvarchive/riscv-v-spec RVV extends a base scalar RISC-V ISA with 32 vector registers and seven unprivileged control-status registers (CSRs) to control the engine. Each vector register could be up to 2^16 bits in length, depending on implementation. - Detect the extension during boot time ("v" letter in the ISA string) - Implement RVV management code in the machine-dependent interfaces that handle CPU and thread state - Add memory-management code for vector state save area. The allocation for save area in thread's PCBs has to be dynamic as the length of registers varies across implementations - Save and restore RVV state on context-switch, fork(), scheduler entry - Enable the extension usage on the first instruction trap from userspace ("lazy" enable) - Implement ucontext.h POSIX API Differential Revision: https://reviews.freebsd.org/D56414 --- sys/conf/files.riscv | 1 + sys/riscv/include/elf.h | 1 + sys/riscv/include/md_var.h | 1 + sys/riscv/include/pcb.h | 10 +- sys/riscv/include/riscvreg.h | 8 +- sys/riscv/include/ucontext.h | 22 +++- sys/riscv/include/vector.h | 36 +++++++ sys/riscv/riscv/exec_machdep.c | 187 ++++++++++++++++++++++++++++++-- sys/riscv/riscv/identcpu.c | 11 +- sys/riscv/riscv/machdep.c | 3 +- sys/riscv/riscv/swtch.S | 55 +++++++++- sys/riscv/riscv/trap.c | 15 ++- sys/riscv/riscv/vector.c | 240 +++++++++++++++++++++++++++++++++++++++++ sys/riscv/riscv/vm_machdep.c | 14 ++- 14 files changed, 587 insertions(+), 17 deletions(-) diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv index 58a31b5f326e..307276b5fd52 100644 --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -97,6 +97,7 @@ riscv/riscv/trap.c standard riscv/riscv/timer.c standard riscv/riscv/uio_machdep.c standard riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack +riscv/riscv/vector.c standard riscv/riscv/vm_machdep.c standard riscv/vmm/vmm.c optional vmm riscv/vmm/vmm_aplic.c optional vmm diff --git a/sys/riscv/include/elf.h b/sys/riscv/include/elf.h index 78788abe1e57..aa5c1318f088 100644 --- a/sys/riscv/include/elf.h +++ b/sys/riscv/include/elf.h @@ -81,6 +81,7 @@ __ElfType(Auxinfo); #define HWCAP_ISA_D HWCAP_ISA_BIT('d') #define HWCAP_ISA_C HWCAP_ISA_BIT('c') #define HWCAP_ISA_H HWCAP_ISA_BIT('h') +#define HWCAP_ISA_V HWCAP_ISA_BIT('v') #define HWCAP_ISA_G \ (HWCAP_ISA_I | HWCAP_ISA_M | HWCAP_ISA_A | HWCAP_ISA_F | HWCAP_ISA_D) #define HWCAP_ISA_B HWCAP_ISA_BIT('b') diff --git a/sys/riscv/include/md_var.h b/sys/riscv/include/md_var.h index 85a51e30b4a7..5f921ed500bf 100644 --- a/sys/riscv/include/md_var.h +++ b/sys/riscv/include/md_var.h @@ -46,6 +46,7 @@ extern bool has_hyp; extern bool has_sstc; extern bool has_sscofpmf; extern bool has_svpbmt; +extern bool has_vector; struct dumperinfo; struct minidumpstate; diff --git a/sys/riscv/include/pcb.h b/sys/riscv/include/pcb.h index bb88516a1385..c0d70d9ab107 100644 --- a/sys/riscv/include/pcb.h +++ b/sys/riscv/include/pcb.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -51,6 +51,14 @@ struct pcb { #define PCB_FP_STARTED 0x1 #define PCB_FP_USERMASK 0x1 vm_offset_t pcb_onfault; /* Copyinout fault handler */ + /* Vector state. */ + uint64_t pcb_vsflags; +#define PCB_VS_STARTED 0x1 + uint64_t pcb_vstart; + uint64_t pcb_vl; + uint64_t pcb_vtype; + uint64_t pcb_vcsr; + void *pcb_vsaved; /* Vector registers area. */ }; #ifdef _KERNEL diff --git a/sys/riscv/include/riscvreg.h b/sys/riscv/include/riscvreg.h index 23feb419d04c..8a81e5e6c2e4 100644 --- a/sys/riscv/include/riscvreg.h +++ b/sys/riscv/include/riscvreg.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2024 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -64,6 +64,12 @@ #define SSTATUS_SPIE_SHIFT 5 #define SSTATUS_SPP (1 << 8) #define SSTATUS_SPP_SHIFT 8 +#define SSTATUS_VS_SHIFT 9 +#define SSTATUS_VS_OFF (0x0 << SSTATUS_VS_SHIFT) +#define SSTATUS_VS_INITIAL (0x1 << SSTATUS_VS_SHIFT) +#define SSTATUS_VS_CLEAN (0x2 << SSTATUS_VS_SHIFT) +#define SSTATUS_VS_DIRTY (0x3 << SSTATUS_VS_SHIFT) +#define SSTATUS_VS_MASK (0x3 << SSTATUS_VS_SHIFT) #define SSTATUS_FS_SHIFT 13 #define SSTATUS_FS_OFF (0x0 << SSTATUS_FS_SHIFT) #define SSTATUS_FS_INITIAL (0x1 << SSTATUS_FS_SHIFT) diff --git a/sys/riscv/include/ucontext.h b/sys/riscv/include/ucontext.h index 13242c457afe..cf6a0bc1ff16 100644 --- a/sys/riscv/include/ucontext.h +++ b/sys/riscv/include/ucontext.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -54,13 +54,31 @@ struct fpregs { int pad; }; +struct riscv_reg_context { + __uint32_t ctx_id; + __uint32_t ctx_size; +}; + +#define RISCV_CTX_MAGIC_END 0 +#define RISCV_CTX_MAGIC_VS 0x53465457 + +struct vector_context { + struct riscv_reg_context ctx; + uint64_t vs_vstart; + uint64_t vs_vl; + uint64_t vs_vtype; + uint64_t vs_vcsr; + uint64_t reserved[3]; +}; + struct __mcontext { struct gpregs mc_gpregs; struct fpregs mc_fpregs; int mc_flags; #define _MC_FP_VALID 0x1 /* Set when mc_fpregs has valid data */ int mc_pad; - __uint64_t mc_spare[8]; /* Space for expansion */ + __uint64_t mc_ptr; /* Address of ctx headers and data */ + __uint64_t mc_spare[7]; /* Space for expansion */ }; typedef struct __mcontext mcontext_t; diff --git a/sys/riscv/include/vector.h b/sys/riscv/include/vector.h new file mode 100644 index 000000000000..a998598c61b6 --- /dev/null +++ b/sys/riscv/include/vector.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2026 Ruslan Bukin <[email protected]> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_VECTOR_H_ +#define _MACHINE_VECTOR_H_ + +void vector_state_init(struct thread *td); +void vector_state_store(struct thread *td); +void vector_state_store_savectx(struct pcb *p); +void vector_state_restore(struct thread *td); +void vector_copy_thread(struct thread *td1, struct thread *td2); +int vector_get_size(void); + +#endif /* !_MACHINE_VECTOR_H_ */ diff --git a/sys/riscv/riscv/exec_machdep.c b/sys/riscv/riscv/exec_machdep.c index 4e0d1f2a7e5c..bf5488249d5c 100644 --- a/sys/riscv/riscv/exec_machdep.c +++ b/sys/riscv/riscv/exec_machdep.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2014 Andrew Turner - * Copyright (c) 2015-2017 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -62,6 +62,8 @@ #include <machine/riscvreg.h> #include <machine/sbi.h> #include <machine/trap.h> +#include <machine/vector.h> +#include <machine/md_var.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -71,6 +73,10 @@ static void get_fpcontext(struct thread *td, mcontext_t *mcp); static void set_fpcontext(struct thread *td, mcontext_t *mcp); +#define CTX_SIZE_VS(buf_size) \ + roundup2(sizeof(struct vector_context) + (buf_size), \ + _Alignof(struct vector_context)) + _Static_assert(sizeof(mcontext_t) == 864, "mcontext_t size incorrect"); _Static_assert(sizeof(ucontext_t) == 936, "ucontext_t size incorrect"); _Static_assert(sizeof(siginfo_t) == 80, "siginfo_t size incorrect"); @@ -227,10 +233,55 @@ get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) return (0); } +static int +restore_vector_state(struct pcb *pcb, struct riscv_reg_context *ctx, + vm_offset_t addr) +{ + struct vector_context vs_ctx; + size_t buf_size; + int error; + + /* Ensure vector engine present. */ + if (!has_vector) + return (EINVAL); + + /* Ensure vector state is initialized. */ + if (pcb->pcb_vsaved == NULL) + return (EINVAL); + + buf_size = vector_get_size(); + if (ctx->ctx_size != CTX_SIZE_VS(buf_size)) + return (EINVAL); + + /* Copy the vector registers. */ + error = copyin((const void *)addr, &vs_ctx, sizeof(vs_ctx)); + if (error != 0) + return (error); + + /* Copy the vector data. */ + if (copyin((void *)(addr + sizeof(vs_ctx)), pcb->pcb_vsaved, + buf_size) != 0) + return (EINVAL); + + /* Restore pcb registers. */ + pcb->pcb_vstart = vs_ctx.vs_vstart; + pcb->pcb_vl = vs_ctx.vs_vl; + pcb->pcb_vtype = vs_ctx.vs_vtype; + pcb->pcb_vcsr = vs_ctx.vs_vcsr; + pcb->pcb_vsflags |= PCB_VS_STARTED; + + return (0); +} + int set_mcontext(struct thread *td, mcontext_t *mcp) { struct trapframe *tf; + struct riscv_reg_context ctx; + struct pcb *pcb; + vm_offset_t addr; + int error, seen_types; + bool done; tf = td->td_frame; @@ -256,8 +307,48 @@ set_mcontext(struct thread *td, mcontext_t *mcp) tf->tf_gp = mcp->mc_gpregs.gp_gp; tf->tf_sepc = mcp->mc_gpregs.gp_sepc; tf->tf_sstatus = mcp->mc_gpregs.gp_sstatus; + set_fpcontext(td, mcp); + if (mcp->mc_ptr == 0) + return (0); + + /* Read any register contexts we find */ + addr = mcp->mc_ptr; + pcb = td->td_pcb; + +#define CTX_TYPE_VS (1 << 0) + + seen_types = 0; + done = false; + do { + if (!__is_aligned(addr, _Alignof(struct riscv_reg_context))) + return (EINVAL); + + error = copyin((const void *)addr, &ctx, sizeof(ctx)); + if (error != 0) + return (error); + + switch (ctx.ctx_id) { + case RISCV_CTX_MAGIC_VS: + if ((seen_types & CTX_TYPE_VS) != 0) + return (EINVAL); + seen_types |= CTX_TYPE_VS; + error = restore_vector_state(pcb, &ctx, addr); + if (error) + return (EINVAL); + break; + case RISCV_CTX_MAGIC_END: + done = true; + break; + default: + return (EINVAL); + } + addr += ctx.ctx_size; + } while (!done); + +#undef CTX_TYPE_VS + return (0); } @@ -333,6 +424,71 @@ sys_sigreturn(struct thread *td, struct sigreturn_args *uap) return (EJUSTRETURN); } +static bool +sendsig_ctx_end(struct thread *td, vm_offset_t *addrp) +{ + struct riscv_reg_context end_ctx; + vm_offset_t ctx_addr; + + *addrp -= sizeof(end_ctx); + ctx_addr = *addrp; + + memset(&end_ctx, 0, sizeof(end_ctx)); + end_ctx.ctx_id = RISCV_CTX_MAGIC_END; + end_ctx.ctx_size = sizeof(end_ctx); + + if (copyout(&end_ctx, (void *)ctx_addr, sizeof(end_ctx)) != 0) + return (false); + return (true); +} + +static bool +sendsig_ctx_vector(struct thread *td, vm_offset_t *addrp) +{ + struct vector_context vs_ctx; + struct pcb *pcb; + size_t buf_size, ctx_size; + vm_offset_t vs_ctx_addr; + + pcb = td->td_pcb; + /* Do nothing if vector hasn't started */ + if (pcb->pcb_vsaved == NULL) + return (true); + + MPASS(pcb->pcb_vsaved != NULL); + + buf_size = vector_get_size(); + ctx_size = CTX_SIZE_VS(buf_size); + + /* Address for the full context. */ + *addrp -= ctx_size; + vs_ctx_addr = *addrp; + + memset(&vs_ctx, 0, sizeof(vs_ctx)); + vs_ctx.ctx.ctx_id = RISCV_CTX_MAGIC_VS; + vs_ctx.ctx.ctx_size = ctx_size; + vs_ctx.vs_vstart = pcb->pcb_vstart; + vs_ctx.vs_vl = pcb->pcb_vl; + vs_ctx.vs_vtype = pcb->pcb_vtype; + vs_ctx.vs_vcsr = pcb->pcb_vcsr; + + /* Copy out the header and data */ + if (copyout(&vs_ctx, (void *)vs_ctx_addr, sizeof(vs_ctx)) != 0) + return (false); + if (copyout(pcb->pcb_vsaved, (void *)(vs_ctx_addr + sizeof(vs_ctx)), + buf_size) != 0) + return (false); + + return (true); +} + +typedef bool(*ctx_func)(struct thread *, vm_offset_t *); +static const ctx_func ctx_funcs[] = { + sendsig_ctx_end, /* Must go first. */ + sendsig_ctx_vector, + NULL, +}; + void sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { @@ -342,8 +498,10 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) struct sigacts *psp; struct thread *td; struct proc *p; + vm_offset_t addr; int onstack; int sig; + int i; td = curthread; p = td->td_proc; @@ -362,16 +520,12 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { - fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + + addr = ((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); } else { - fp = (struct sigframe *)td->td_frame->tf_sp; + addr = td->td_frame->tf_sp; } - /* Make room, keeping the stack aligned */ - fp--; - fp = STACKALIGN(fp); - /* Fill in the frame to copy out */ bzero(&frame, sizeof(frame)); get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); @@ -383,6 +537,25 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); + for (i = 0; ctx_funcs[i] != NULL; i++) { + if (!ctx_funcs[i](td, &addr)) { + CTR4(KTR_SIG, + "sendsig: frame sigexit td=%p fp=%#lx func[%d]=%p", + td, addr, i, ctx_funcs[i]); + PROC_LOCK(p); + sigexit(td, SIGILL); + /* NOTREACHED */ + } + } + + /* Point at the first context */ + frame.sf_uc.uc_mcontext.mc_ptr = addr; + + /* Make room, keeping the stack aligned */ + fp = (struct sigframe *)addr; + fp--; + fp = (struct sigframe *)STACKALIGN(fp); + /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ diff --git a/sys/riscv/riscv/identcpu.c b/sys/riscv/riscv/identcpu.c index af71cc2f89f0..7b2d9a1d1259 100644 --- a/sys/riscv/riscv/identcpu.c +++ b/sys/riscv/riscv/identcpu.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * - * Copyright (c) 2015-2016 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * Copyright (c) 2022 Mitchell Horne <[email protected]> * Copyright (c) 2023 The FreeBSD Foundation @@ -75,6 +75,7 @@ u_int mmu_caps; /* Supervisor-mode extension support. */ bool has_hyp; +bool has_vector; bool __read_frequently has_sstc; bool __read_frequently has_sscofpmf; bool has_svpbmt; @@ -280,6 +281,7 @@ parse_riscv_isa(struct cpu_desc *desc, char *isa, int len) case 'h': case 'i': case 'm': + case 'v': desc->isa_extensions |= HWCAP_ISA_BIT(isa[i]); i++; break; @@ -402,6 +404,9 @@ identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc) return; } + if (bootverbose) + printf("hart %d isa: %s\n", hart, isa); + /* * The string is specified to be lowercase, but let's be * certain. @@ -462,6 +467,7 @@ update_global_capabilities(u_int cpu, struct cpu_desc *desc) UPDATE_CAP(mmu_caps, desc->mmu_caps); /* Supervisor-mode extension support. */ + UPDATE_CAP(has_vector, (desc->isa_extensions & HWCAP_ISA_V) != 0); UPDATE_CAP(has_hyp, (desc->isa_extensions & HWCAP_ISA_H) != 0); UPDATE_CAP(has_sstc, (desc->smode_extensions & SV_SSTC) != 0); UPDATE_CAP(has_sscofpmf, (desc->smode_extensions & SV_SSCOFPMF) != 0); @@ -605,7 +611,8 @@ printcpuinfo(u_int cpu) "\04Double" "\06Float" "\10Hypervisor" - "\15Mult/Div"); + "\15Mult/Div" + "\26Vector"); } if (SHOULD_PRINT(smode_extensions)) { diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c index f5479c479109..99cede3efba6 100644 --- a/sys/riscv/riscv/machdep.c +++ b/sys/riscv/riscv/machdep.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2014 Andrew Turner - * Copyright (c) 2015-2017 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -299,6 +299,7 @@ init_proc0(void *kstack) thread0.td_kstack_pages = KSTACK_PAGES; thread0.td_pcb = &pcb0; thread0.td_pcb->pcb_fpflags = 0; + thread0.td_pcb->pcb_vsflags = 0; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; } diff --git a/sys/riscv/riscv/swtch.S b/sys/riscv/riscv/swtch.S index fc9b493744b8..b7b2e1ec9dd3 100644 --- a/sys/riscv/riscv/swtch.S +++ b/sys/riscv/riscv/swtch.S @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2017 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -210,6 +210,17 @@ ENTRY(cpu_throw) call _C_LABEL(pmap_activate_sw) mv a0, s0 + /* Is VS enabled for new thread? */ + ld t0, TD_FRAME(a0) + ld t1, (TF_SSTATUS)(t0) + li t2, SSTATUS_VS_MASK + and t3, t1, t2 + beqz t3, 1f /* No, skip. */ + + call _C_LABEL(vector_state_restore) + mv a0, s0 + +1: /* Store the new curthread */ sd a0, PC_CURTHREAD(tp) /* And the new pcb */ @@ -304,6 +315,33 @@ ENTRY(cpu_switch) __fpe_state_save x13 1: + /* + * Is VS enabled and is it in dirty state + * for the old thread? + */ + ld t0, TD_FRAME(a0) + ld t1, (TF_SSTATUS)(t0) + li t2, SSTATUS_VS_MASK + and t3, t1, t2 + li t2, SSTATUS_VS_DIRTY + bne t3, t2, 1f /* No, skip. */ + + /* Yes, mark VS clean and save registers. */ + li t2, ~SSTATUS_VS_MASK + and t3, t1, t2 + li t2, SSTATUS_VS_CLEAN + or t3, t3, t2 + sd t3, (TF_SSTATUS)(t0) + + mv s0, a0 + mv s1, a1 + mv s2, a2 + call _C_LABEL(vector_state_store) + mv a0, s0 + mv a1, s1 + mv a2, s2 +1: + /* Activate the new thread's pmap */ mv s0, a0 mv s1, a1 @@ -321,6 +359,20 @@ ENTRY(cpu_switch) ld t0, TD_LOCK(a1) beq t0, s2, 1b #endif + + /* Is VS enabled for new thread? */ + ld t0, TD_FRAME(a1) + ld t1, (TF_SSTATUS)(t0) + li t2, SSTATUS_VS_MASK + and t3, t1, t2 + beqz t3, 1f /* No, skip. */ + + /* Restore vector registers. */ + mv a0, a1 + call _C_LABEL(vector_state_restore) + mv a1, s1 +1: + /* * Restore the saved context. */ @@ -453,5 +505,6 @@ ENTRY(savectx) sd s11, (PCB_S + 11 * 8)(a0) __fpe_state_save a0 + call _C_LABEL(vector_state_store_savectx) ret END(savectx) diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c index d6df1245a24e..4ca3e031670e 100644 --- a/sys/riscv/riscv/trap.c +++ b/sys/riscv/riscv/trap.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2018 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -62,6 +62,8 @@ #include <machine/frame.h> #include <machine/pcb.h> #include <machine/pcpu.h> +#include <machine/vector.h> +#include <machine/md_var.h> #include <machine/resource.h> @@ -443,6 +445,17 @@ do_trap_user(struct trapframe *frame) pcb->pcb_fpflags |= PCB_FP_STARTED; break; } + if (has_vector && (pcb->pcb_vsflags & PCB_VS_STARTED) == 0) { + /* + * Could be a vector trap. Enable VS usage + * for this thread and try again. + */ + vector_state_init(td); + frame->tf_sstatus &= ~SSTATUS_VS_MASK; + frame->tf_sstatus |= SSTATUS_VS_CLEAN; + pcb->pcb_vsflags |= PCB_VS_STARTED; + break; + } call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc, exception); userret(td, frame); diff --git a/sys/riscv/riscv/vector.c b/sys/riscv/riscv/vector.c new file mode 100644 index 000000000000..9c57cefd5a4b --- /dev/null +++ b/sys/riscv/riscv/vector.c @@ -0,0 +1,240 @@ +/*- + * Copyright (c) 2026 Ruslan Bukin <[email protected]> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/eventhandler.h> +#include <sys/kernel.h> +#include <sys/proc.h> + +#include <vm/uma.h> + +#include <machine/vector.h> +#include <machine/pcb.h> +#include <machine/reg.h> +#include <machine/md_var.h> + +static MALLOC_DEFINE(M_RVV_CTX, "rvv_ctx", "Contexts for user Vector state"); + +static void +vector_enable(void) +{ + uint64_t reg; + + reg = SSTATUS_VS_CLEAN; + + csr_set(sstatus, reg); +} + +static void +vector_disable(void) +{ + uint64_t mask; + + mask = SSTATUS_VS_MASK; + + csr_clear(sstatus, mask); +} + +static void +vector_state_store_common(struct pcb *p, void *datap) +{ + uint64_t vl; + + vector_enable(); + + /* Store vector CSRs. */ + __asm __volatile( + "csrr %0, vstart\n\t" + "csrr %1, vtype\n\t" + "csrr %2, vl\n\t" + "csrr %3, vcsr\n\t" + : "=r" (p->pcb_vstart), "=r" (p->pcb_vtype), + "=r" (p->pcb_vl), "=r" (p->pcb_vcsr) ::); + + if (datap == NULL) + goto done; + + /* Store vector registers. */ + __asm __volatile( + ".option push\n\t" + ".option arch, +zve32x\n\t" + "vsetvli %0, x0, e8, m8, ta, ma\n\t" + "vse8.v v0, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v8, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v16, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v24, (%1)\n\t" + ".option pop\n\t" + : "=&r" (vl) : "r" (datap) : "memory"); + +done: + vector_disable(); +} + +void +vector_state_store_savectx(struct pcb *p) +{ + + if (!has_vector) + return; + + /* + * savectx() will be called on panic with dumppcb as an argument. + * Save the vector CSR registers, but ignore the vector data. + */ + + vector_state_store_common(p, NULL); +} + +void +vector_state_store(struct thread *td) +{ + struct pcb *p; + void *datap; + + p = td->td_pcb; + + KASSERT(p->pcb_vsaved != NULL, ("VS area is NULL")); + + datap = p->pcb_vsaved; + + vector_state_store_common(p, datap); +} + +void +vector_state_restore(struct thread *td) +{ + struct pcb *p; + void *datap; + uint64_t vl; + + p = td->td_pcb; + + KASSERT(p->pcb_vsaved != NULL, ("VS area is NULL")); + + datap = p->pcb_vsaved; + + vector_enable(); + + /* Restore vector registers. */ + __asm __volatile( + ".option push\n\t" + ".option arch, +zve32x\n\t" + "vsetvli %0, x0, e8, m8, ta, ma\n\t" + "vle8.v v0, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v8, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v16, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v24, (%1)\n\t" + ".option pop\n\t" + : "=&r" (vl) : "r" (datap) : "memory"); + + /* Restore vector CSRs. */ + __asm __volatile( + ".option push\n\t" + ".option arch, +zve32x\n\t" + "vsetvl x0, %2, %1\n\t" + ".option pop\n\t" + "csrw vstart, %0\n\t" + "csrw vcsr, %3\n\t" + :: "r" (p->pcb_vstart), "r" (p->pcb_vtype), + "r" (p->pcb_vl), "r" (p->pcb_vcsr)); + + vector_disable(); +} + +int +vector_get_size(void) +{ + int len; + + /* RISC-V has 32 vector registers of vlenb size each. */ + + vector_enable(); + len = csr_read(vlenb) * 32; + vector_disable(); + + return (len); +} + +void +vector_state_init(struct thread *td) +{ + struct pcb *p; + int len; + + p = td->td_pcb; + + KASSERT(p->pcb_vsaved == NULL, ("vsaved is already initialized")); + + len = vector_get_size(); + + p->pcb_vsaved = malloc(len, M_RVV_CTX, M_WAITOK | M_ZERO); +} + +void +vector_copy_thread(struct thread *td1, struct thread *td2) +{ + struct pcb *p1; + struct pcb *p2; + int len; + + /* Struct pcb already copied, now init the vector save area. */ + + p1 = td1->td_pcb; + p2 = td2->td_pcb; + p2->pcb_vsaved = NULL; + + vector_state_init(td2); + + len = vector_get_size(); + + memcpy(p2->pcb_vsaved, p1->pcb_vsaved, len); +} + +static void +vector_thread_dtor(void *arg __unused, struct thread *td) +{ + void *datap; + + datap = td->td_pcb->pcb_vsaved; + + free(datap, M_RVV_CTX); +} + +static void +vector_init(const void *dummy __unused) +{ + + EVENTHANDLER_REGISTER(thread_dtor, vector_thread_dtor, NULL, + EVENTHANDLER_PRI_ANY); +} + +SYSINIT(vector, SI_SUB_SMP, SI_ORDER_ANY, vector_init, NULL); diff --git a/sys/riscv/riscv/vm_machdep.c b/sys/riscv/riscv/vm_machdep.c index e538921c9f60..b3e4100ebddb 100644 --- a/sys/riscv/riscv/vm_machdep.c +++ b/sys/riscv/riscv/vm_machdep.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2018 Ruslan Bukin <[email protected]> + * Copyright (c) 2015-2026 Ruslan Bukin <[email protected]> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -53,6 +53,7 @@ #include <machine/pcb.h> #include <machine/frame.h> #include <machine/sbi.h> +#include <machine/vector.h> #if __riscv_xlen == 64 #define TP_OFFSET 16 /* sizeof(struct tcb) */ @@ -99,6 +100,14 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) critical_exit(); } + /* Ensure the Vector state is saved before copying the pcb. */ + if ((td1->td_pcb->pcb_vsflags & PCB_VS_STARTED) != 0) { + MPASS(td1 == curthread); + critical_enter(); + vector_state_store(td1); + critical_exit(); + } + pcb2 = td2->td_pcb; bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); @@ -120,6 +129,9 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) td2->td_pcb->pcb_ra = (uintptr_t)fork_trampoline; td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame; + if ((td1->td_pcb->pcb_vsflags & PCB_VS_STARTED) != 0) + vector_copy_thread(td1, td2); *** 4 LINES SKIPPED ***
