The branch main has been updated by andrew: URL: https://cgit.FreeBSD.org/src/commit/?id=dd08cb0c8d6e1737d2dddf978a0291f09911a715
commit dd08cb0c8d6e1737d2dddf978a0291f09911a715 Author: Kajetan Puchalski <[email protected]> AuthorDate: 2026-07-06 14:16:34 +0000 Commit: Andrew Turner <[email protected]> CommitDate: 2026-07-06 15:18:29 +0000 arm64/vmm: Use the VNCR_EL2 memory page to store guest registers Wherever possible, move the storage space for guest register values from the hypctx struct into a preallocated memory page matching the layout of the page pointed to by VNCR_EL2. This will streamline implementing support for nested virtualization, but the implementation itself is not reliant on the presence of nested virtualization architecture features. Signed-off-by: Kajetan Puchalski <[email protected]> Reviewed by: andrew Sponsored by: Arm Ltd Differential Revision: https://reviews.freebsd.org/D56551 --- sys/arm64/include/hypervisor.h | 2 + sys/arm64/vmm/arm64.h | 168 +++++++++++++++++++++++++++++++++++------ sys/arm64/vmm/vmm_arm64.c | 78 ++++++++++++------- sys/arm64/vmm/vmm_hyp.c | 111 +++++++++++++++++---------- sys/arm64/vmm/vmm_nvhe.c | 3 + sys/arm64/vmm/vmm_reset.c | 22 +----- 6 files changed, 277 insertions(+), 107 deletions(-) diff --git a/sys/arm64/include/hypervisor.h b/sys/arm64/include/hypervisor.h index bc5b6f4344ee..0b6492785a0a 100644 --- a/sys/arm64/include/hypervisor.h +++ b/sys/arm64/include/hypervisor.h @@ -2185,6 +2185,8 @@ /* Assumed to be 0 by locore.S */ #define VTTBR_HOST 0x0000000000000000 +/* Size of the memory page pointed to by VNCR_EL2 */ +#define VNCR_PAGE_SIZE 4096 /* * Memory offsets of registers redirected to memory by HCR_EL2.NV2 * relative to the base address stored in VNCR_EL2. diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h index 6062d1c07ae4..4db26709f21e 100644 --- a/sys/arm64/vmm/arm64.h +++ b/sys/arm64/vmm/arm64.h @@ -39,6 +39,109 @@ struct vgic_v3; struct vgic_v3_cpu; +/* + Encode the index of a given register within VNCR into the enum value. + We move the indices up by VNCR_START to not interfere with values for + non-VNCR registers. +*/ +#define VNCR_REG(reg) reg = (VNCR_START + VNCR_##reg / 8) +/* Retrieve the VNCR offset from the enum value */ +#define REG_VNCR_OFFSET(val) ((val - VNCR_START) * 8) + +enum hypctx_sysreg { + /* VNCR Registers */ + VNCR_START, + + VNCR_REG(VTTBR_EL2), + VNCR_REG(VSTTBR_EL2), + VNCR_REG(VTCR_EL2), + VNCR_REG(VSTCR_EL2), + VNCR_REG(VMPIDR_EL2), /* Virtualization Multiprocessor ID Register */ + VNCR_REG(CNTVOFF_EL2), + VNCR_REG(HCR_EL2), /* Hypervisor Configuration Register */ + VNCR_REG(HSTR_EL2), + VNCR_REG(VPIDR_EL2), /* Virtualization Processor ID Register */ + VNCR_REG(TPIDR_EL2), + VNCR_REG(HCRX_EL2), /* Extended Hypervisor Configuration Register */ + VNCR_REG(VNCR_EL2), + VNCR_REG(CPACR_EL1), /* Architectural Feature Access Control Register */ + VNCR_REG(CONTEXTIDR_EL1), /* Current Process Identifier */ + VNCR_REG(SCTLR_EL1), /* System Control Register */ + VNCR_REG(ACTLR_EL1), /* Auxiliary Control Register */ + VNCR_REG(TCR_EL1), /* Translation Control Register */ + VNCR_REG(AFSR0_EL1), /* Auxiliary Fault Status Register 0 */ + VNCR_REG(AFSR1_EL1), /* Auxiliary Fault Status Register 1 */ + VNCR_REG(ESR_EL1), /* Exception Syndrome Register */ + VNCR_REG(MAIR_EL1), /* Memory Attribute Indirection Register */ + VNCR_REG(AMAIR_EL1), /* Auxiliary Memory Attribute Indirection Register */ + VNCR_REG(MDSCR_EL1), /* Monitor Debug System Control Register */ + VNCR_REG(SPSR_EL1), /* Saved Program Status Register */ + VNCR_REG(CNTV_CVAL_EL0), + VNCR_REG(CNTV_CTL_EL0), + VNCR_REG(CNTP_CVAL_EL0), + VNCR_REG(CNTP_CTL_EL0), + VNCR_REG(SCXTNUM_EL1), + VNCR_REG(TFSR_EL1), + VNCR_REG(HDFGRTR2_EL2), + VNCR_REG(CNTPOFF_EL2), + VNCR_REG(HDFGWTR2_EL2), + VNCR_REG(HFGRTR_EL2), + VNCR_REG(HFGWTR_EL2), + VNCR_REG(HFGITR_EL2), + VNCR_REG(HDFGRTR_EL2), + VNCR_REG(HDFGWTR_EL2), + VNCR_REG(ZCR_EL1), + VNCR_REG(HAFGRTR_EL2), + VNCR_REG(SMCR_EL1), + VNCR_REG(SMPRIMAP_EL2), + VNCR_REG(TTBR0_EL1), /* Translation Table Base Register 0 */ + VNCR_REG(TTBR1_EL1), /* Translation Table Base Register 1 */ + VNCR_REG(FAR_EL1), /* Fault Address Register */ + VNCR_REG(ELR_EL1), /* Exception Link Register */ + VNCR_REG(SP_EL1), + VNCR_REG(VBAR_EL1), /* Vector Base Address Register */ + VNCR_REG(TCR2_EL1), /* Translation Control Register 2 */ + VNCR_REG(SCTLR2_EL1), + VNCR_REG(MAIR2_EL1), + VNCR_REG(AMAIR2_EL1), + VNCR_REG(PIRE0_EL1), + VNCR_REG(PIRE0_EL2), + VNCR_REG(PIR_EL1), + VNCR_REG(POR_EL1), + VNCR_REG(S2PIR_EL2), + VNCR_REG(S2POR_EL1), + VNCR_REG(HFGRTR2_EL2), + VNCR_REG(HFGWTR2_EL2), + VNCR_REG(PFAR_EL1), + VNCR_REG(HFGITR2_EL2), + VNCR_REG(SCTLRMASK_EL1), + VNCR_REG(CPACRMASK_EL1), + VNCR_REG(SCTLR2MASK_EL1), + VNCR_REG(TCRMASK_EL1), + VNCR_REG(TCR2MASK_EL1), + VNCR_REG(ACTLRMASK_EL1), + VNCR_REG(ICH_HCR_EL2), + VNCR_REG(ICH_VMCR_EL2), + VNCR_REG(VDISR_EL2), + VNCR_REG(VSESR_EL2), + VNCR_REG(PMBLIMITR_EL1), + VNCR_REG(PMBPTR_EL1), + VNCR_REG(PMBSR_EL1), + VNCR_REG(PMSCR_EL1), + VNCR_REG(PMSEVFR_EL1), + VNCR_REG(PMSICR_EL1), + VNCR_REG(PMSIRR_EL1), + VNCR_REG(PMSLATFR_EL1), + VNCR_REG(PMSNEVFR_EL1), + VNCR_REG(PMSDSFR_EL1), + VNCR_REG(TRFCR_EL1), + VNCR_REG(TRCITECR_EL1), + VNCR_REG(GCSPR_EL1), + VNCR_REG(GCSCR_EL1), + VNCR_REG(BRBCR_EL1), + VNCR_REG(SPMACCESSR_EL1), +}; + /* * Per-vCPU hypervisor state. */ @@ -46,34 +149,15 @@ struct hypctx { struct trapframe tf; /* - * EL1 control registers. + * EL1 & EL0 registers. */ - uint64_t elr_el1; /* Exception Link Register */ uint64_t sp_el0; /* Stack pointer */ uint64_t tpidr_el0; /* EL0 Software ID Register */ uint64_t tpidrro_el0; /* Read-only Thread ID Register */ uint64_t tpidr_el1; /* EL1 Software ID Register */ - uint64_t vbar_el1; /* Vector Base Address Register */ - - uint64_t actlr_el1; /* Auxiliary Control Register */ - uint64_t afsr0_el1; /* Auxiliary Fault Status Register 0 */ - uint64_t afsr1_el1; /* Auxiliary Fault Status Register 1 */ - uint64_t amair_el1; /* Auxiliary Memory Attribute Indirection Register */ - uint64_t contextidr_el1; /* Current Process Identifier */ - uint64_t cpacr_el1; /* Architectural Feature Access Control Register */ uint64_t csselr_el1; /* Cache Size Selection Register */ - uint64_t esr_el1; /* Exception Syndrome Register */ - uint64_t far_el1; /* Fault Address Register */ - uint64_t mair_el1; /* Memory Attribute Indirection Register */ uint64_t mdccint_el1; /* Monitor DCC Interrupt Enable Register */ - uint64_t mdscr_el1; /* Monitor Debug System Control Register */ uint64_t par_el1; /* Physical Address Register */ - uint64_t sctlr_el1; /* System Control Register */ - uint64_t tcr_el1; /* Translation Control Register */ - uint64_t tcr2_el1; /* Translation Control Register 2 */ - uint64_t ttbr0_el1; /* Translation Table Base Register 0 */ - uint64_t ttbr1_el1; /* Translation Table Base Register 1 */ - uint64_t spsr_el1; /* Saved Program Status Register */ uint64_t pmcr_el0; /* Performance Monitors Control Register */ uint64_t pmccntr_el0; @@ -100,6 +184,8 @@ struct hypctx { uint64_t mdcr_el2; /* Monitor Debug Configuration Register */ uint64_t vpidr_el2; /* Virtualization Processor ID Register */ uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */ + /* On systems without NV2 this still points to the register storage memory page */ + uint64_t vncr_el2; /* Virtual Nested Control Register */ /* FEAT_FGT registers */ /*uint64_t hafgrtr_el2; *//* For FEAT_AMUv1 (not supported) */ @@ -116,7 +202,6 @@ struct hypctx { uint64_t hfgrtr2_el2; uint64_t hfgwtr2_el2; - uint64_t el2_addr; /* The address of this in el2 space */ struct hyp *hyp; struct vcpu *vcpu; struct { @@ -136,8 +221,49 @@ struct hypctx { struct vgic_v3_cpu *vgic_cpu; bool has_exception; bool dbg_oslock; + + /* + * Memory page pointed at by VNCR_EL2. Contains storage for registers + * the accesses to which are redirected to memory by FEAT_NV2. + * NOTE: The storage is still used even if FEAT_NV2 is not present. + */ + void *vncr_regs; + /* Memory page used to store the host's values of VNCR registers. */ + void *host_vncr_regs; + + uint64_t el2_addr; /* The address of this in el2 space */ + uint64_t el2_vncr_addr; /* The address of vncr_regs in el2 space */ + uint64_t el2_host_vncr_addr; }; +/* For non-VHE vmm_hyp.c, this will already be defined in vmm_nvhe.c */ +#ifndef __hypctx_vncr_sysreg +#define __hypctx_vncr_sysreg(hypctx, reg) \ + ((uint64_t *)((char *)hypctx->vncr_regs + REG_VNCR_OFFSET(reg))) +#endif + +static inline uint64_t * +hypctx_sys_reg(struct hypctx *hypctx, int reg /* enum hypctx_sysreg */) +{ + if (reg > VNCR_START) + return (__hypctx_vncr_sysreg(hypctx, reg)); + /* TODO: uniform handling for non-VNCR registers */ + return (NULL); +} + +static inline void +hypctx_write_sys_reg(struct hypctx *hypctx, enum hypctx_sysreg reg, + uint64_t val) +{ + *hypctx_sys_reg(hypctx, reg) = val; +} + +static inline uint64_t +hypctx_read_sys_reg(struct hypctx *hypctx, enum hypctx_sysreg reg) +{ + return (*hypctx_sys_reg(hypctx, reg)); +} + struct hyp { struct vm *vm; struct vtimer vtimer; diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c index 8fe0bf62bf8b..3615d2da66cc 100644 --- a/sys/arm64/vmm/vmm_arm64.c +++ b/sys/arm64/vmm/vmm_arm64.c @@ -490,7 +490,8 @@ el2_hyp_size(struct vm *vm) static vm_size_t el2_hypctx_size(void) { - return (round_page(sizeof(struct hypctx))); + /* Allocation for hypctx, one vncr page & one host state page */ + return (round_page(sizeof(struct hypctx) + 2 * VNCR_PAGE_SIZE)); } static vm_offset_t @@ -559,7 +560,17 @@ vmmops_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid) vm_size_t size; size = el2_hypctx_size(); + /* + * Allocation memory layout: + 0x0000 - 0x0fff <- struct hypctx + 0x1000 - 0x1fff <- VNCR memory page + 0x2000 - 0x2fff <- host_ctx memory page + */ hypctx = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO); + _Static_assert(sizeof(struct hypctx) < VNCR_PAGE_SIZE, + "struct hypctx exceeds VNCR_PAGE_SIZE"); + hypctx->vncr_regs = (void *)((char *)hypctx + VNCR_PAGE_SIZE); + hypctx->host_vncr_regs = (void *)((char *)hypctx + 2 * VNCR_PAGE_SIZE); KASSERT(vcpuid >= 0 && vcpuid < vm_get_maxcpus(hyp->vm), ("%s: Invalid vcpuid %d", __func__, vcpuid)); @@ -574,9 +585,12 @@ vmmops_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid) vtimer_cpuinit(hypctx); vgic_cpuinit(hypctx); - if (!in_vhe()) + if (!in_vhe()) { hypctx->el2_addr = el2_map_enter((vm_offset_t)hypctx, size, VM_PROT_READ | VM_PROT_WRITE); + hypctx->el2_vncr_addr = hypctx->el2_addr + VNCR_PAGE_SIZE; + hypctx->el2_host_vncr_addr = hypctx->el2_addr + 2 * VNCR_PAGE_SIZE; + } return (hypctx); } @@ -639,12 +653,14 @@ arm64_gen_inst_emul_data(struct hypctx *hypctx, uint32_t esr_iss, vie->reg = reg_num; paging = &vme_ret->u.inst_emul.paging; - paging->ttbr0_addr = hypctx->ttbr0_el1 & ~(TTBR_ASID_MASK | TTBR_CnP); - paging->ttbr1_addr = hypctx->ttbr1_el1 & ~(TTBR_ASID_MASK | TTBR_CnP); - paging->tcr_el1 = hypctx->tcr_el1; - paging->tcr2_el1 = hypctx->tcr2_el1; + paging->ttbr0_addr = hypctx_read_sys_reg(hypctx, TTBR0_EL1) & + ~(TTBR_ASID_MASK | TTBR_CnP); + paging->ttbr1_addr = hypctx_read_sys_reg(hypctx, TTBR1_EL1) & + ~(TTBR_ASID_MASK | TTBR_CnP); + paging->tcr_el1 = hypctx_read_sys_reg(hypctx, TCR_EL1); + paging->tcr2_el1 = hypctx_read_sys_reg(hypctx, TCR2_EL1); paging->flags = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32); - if ((hypctx->sctlr_el1 & SCTLR_M) != 0) + if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & SCTLR_M) != 0) paging->flags |= VM_GP_MMU_ENABLED; } @@ -1078,7 +1094,7 @@ fault: int vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) { - uint64_t excp_type; + uint64_t excp_type, vbar_el1; int handled; register_t daif; struct hyp *hyp; @@ -1097,36 +1113,41 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) for (;;) { if (hypctx->has_exception) { hypctx->has_exception = false; - hypctx->elr_el1 = hypctx->tf.tf_elr; + hypctx_write_sys_reg(hypctx, ELR_EL1, hypctx->tf.tf_elr); mode = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32); + vbar_el1 = hypctx_read_sys_reg(hypctx, VBAR_EL1); if (mode == PSR_M_EL1t) { - hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x0; + hypctx->tf.tf_elr = vbar_el1 + 0x0; } else if (mode == PSR_M_EL1h) { - hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x200; + hypctx->tf.tf_elr = vbar_el1 + 0x200; } else if ((mode & PSR_M_32) == PSR_M_64) { /* 64-bit EL0 */ - hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x400; + hypctx->tf.tf_elr = vbar_el1 + 0x400; } else { /* 32-bit EL0 */ - hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x600; + hypctx->tf.tf_elr = vbar_el1 + 0x600; } /* Set the new spsr */ - hypctx->spsr_el1 = hypctx->tf.tf_spsr; + hypctx_write_sys_reg(hypctx, SPSR_EL1, hypctx->tf.tf_spsr); /* Set the new cpsr */ - hypctx->tf.tf_spsr = hypctx->spsr_el1 & PSR_FLAGS; + hypctx->tf.tf_spsr = hypctx_read_sys_reg(hypctx, + SPSR_EL1) & PSR_FLAGS; + hypctx->tf.tf_spsr |= PSR_DAIF | PSR_M_EL1h; /* * Update fields that may change on exeption entry * based on how sctlr_el1 is configured. */ - if ((hypctx->sctlr_el1 & SCTLR_SPAN) == 0) + if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & + SCTLR_SPAN) == 0) hypctx->tf.tf_spsr |= PSR_PAN; - if ((hypctx->sctlr_el1 & SCTLR_DSSBS) == 0) + if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & + SCTLR_DSSBS) == 0) hypctx->tf.tf_spsr &= ~PSR_SSBS; else hypctx->tf.tf_spsr |= PSR_SSBS; @@ -1259,15 +1280,15 @@ hypctx_regptr(struct hypctx *hypctx, int reg) case VM_REG_GUEST_PC: return (&hypctx->tf.tf_elr); case VM_REG_GUEST_SCTLR_EL1: - return (&hypctx->sctlr_el1); + return hypctx_sys_reg(hypctx, SCTLR_EL1); case VM_REG_GUEST_TTBR0_EL1: - return (&hypctx->ttbr0_el1); + return hypctx_sys_reg(hypctx, TTBR0_EL1); case VM_REG_GUEST_TTBR1_EL1: - return (&hypctx->ttbr1_el1); + return hypctx_sys_reg(hypctx, TTBR1_EL1); case VM_REG_GUEST_TCR_EL1: - return (&hypctx->tcr_el1); + return hypctx_sys_reg(hypctx, TCR_EL1); case VM_REG_GUEST_TCR2_EL1: - return (&hypctx->tcr2_el1); + return hypctx_sys_reg(hypctx, TCR2_EL1); case VM_REG_GUEST_MPIDR_EL1: return (&hypctx->vmpidr_el2); default: @@ -1327,8 +1348,8 @@ vmmops_exception(void *vcpui, uint64_t esr, uint64_t far) panic("%s: %s%d is running", __func__, vm_name(hypctx->hyp->vm), vcpu_vcpuid(hypctx->vcpu)); - hypctx->far_el1 = far; - hypctx->esr_el1 = esr; + hypctx_write_sys_reg(hypctx, FAR_EL1, far); + hypctx_write_sys_reg(hypctx, ESR_EL1, esr); hypctx->has_exception = true; return (0); @@ -1382,17 +1403,18 @@ vmmops_setcap(void *vcpui, int num, int val) if (val != 0) { hypctx->debug_spsr |= (hypctx->tf.tf_spsr & PSR_SS); - hypctx->debug_mdscr |= (hypctx->mdscr_el1 & MDSCR_SS); + hypctx->debug_mdscr |= + (hypctx_read_sys_reg(hypctx, MDSCR_EL1) & MDSCR_SS); hypctx->tf.tf_spsr |= PSR_SS; - hypctx->mdscr_el1 |= MDSCR_SS; + *hypctx_sys_reg(hypctx, MDSCR_EL1) |= MDSCR_SS; hypctx->mdcr_el2 |= MDCR_EL2_TDE; } else { hypctx->tf.tf_spsr &= ~PSR_SS; hypctx->tf.tf_spsr |= hypctx->debug_spsr; hypctx->debug_spsr &= ~PSR_SS; - hypctx->mdscr_el1 &= ~MDSCR_SS; - hypctx->mdscr_el1 |= hypctx->debug_mdscr; + *hypctx_sys_reg(hypctx, MDSCR_EL1) &= ~MDSCR_SS; + *hypctx_sys_reg(hypctx, MDSCR_EL1) |= hypctx->debug_mdscr; hypctx->debug_mdscr &= ~MDSCR_SS; if ((hypctx->setcaps & (1ul << VM_CAP_BRK_EXIT)) == 0) hypctx->mdcr_el2 &= ~MDCR_EL2_TDE; diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c index 27fdf18e4700..bde283a077c8 100644 --- a/sys/arm64/vmm/vmm_hyp.c +++ b/sys/arm64/vmm/vmm_hyp.c @@ -246,30 +246,44 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest, hypctx->tpidrro_el0 = READ_SPECIALREG(tpidrro_el0); hypctx->tpidr_el1 = READ_SPECIALREG(tpidr_el1); - hypctx->actlr_el1 = READ_SPECIALREG(actlr_el1); + hypctx_write_sys_reg(hypctx, ACTLR_EL1, READ_SPECIALREG(actlr_el1)); hypctx->csselr_el1 = READ_SPECIALREG(csselr_el1); hypctx->mdccint_el1 = READ_SPECIALREG(mdccint_el1); - hypctx->mdscr_el1 = READ_SPECIALREG(mdscr_el1); + hypctx_write_sys_reg(hypctx, MDSCR_EL1, READ_SPECIALREG(mdscr_el1)); if (guest_or_nonvhe(guest)) { - hypctx->elr_el1 = READ_SPECIALREG(EL1_REG(ELR)); - hypctx->vbar_el1 = READ_SPECIALREG(EL1_REG(VBAR)); - - hypctx->afsr0_el1 = READ_SPECIALREG(EL1_REG(AFSR0)); - hypctx->afsr1_el1 = READ_SPECIALREG(EL1_REG(AFSR1)); - hypctx->amair_el1 = READ_SPECIALREG(EL1_REG(AMAIR)); - hypctx->contextidr_el1 = READ_SPECIALREG(EL1_REG(CONTEXTIDR)); - hypctx->cpacr_el1 = READ_SPECIALREG(EL1_REG(CPACR)); - hypctx->esr_el1 = READ_SPECIALREG(EL1_REG(ESR)); - hypctx->far_el1 = READ_SPECIALREG(EL1_REG(FAR)); - hypctx->mair_el1 = READ_SPECIALREG(EL1_REG(MAIR)); - hypctx->sctlr_el1 = READ_SPECIALREG(EL1_REG(SCTLR)); - hypctx->spsr_el1 = READ_SPECIALREG(EL1_REG(SPSR)); - hypctx->tcr_el1 = READ_SPECIALREG(EL1_REG(TCR)); + hypctx_write_sys_reg(hypctx, ELR_EL1, + READ_SPECIALREG(EL1_REG(ELR))); + hypctx_write_sys_reg(hypctx, VBAR_EL1, + READ_SPECIALREG(EL1_REG(VBAR))); + hypctx_write_sys_reg(hypctx, AFSR0_EL1, + READ_SPECIALREG(EL1_REG(AFSR0))); + hypctx_write_sys_reg(hypctx, AFSR1_EL1, + READ_SPECIALREG(EL1_REG(AFSR1))); + hypctx_write_sys_reg(hypctx, AMAIR_EL1, + READ_SPECIALREG(EL1_REG(AMAIR))); + hypctx_write_sys_reg(hypctx, CONTEXTIDR_EL1, + READ_SPECIALREG(EL1_REG(CONTEXTIDR))); + hypctx_write_sys_reg(hypctx, CPACR_EL1, + READ_SPECIALREG(EL1_REG(CPACR))); + hypctx_write_sys_reg(hypctx, ESR_EL1, + READ_SPECIALREG(EL1_REG(ESR))); + hypctx_write_sys_reg(hypctx, FAR_EL1, + READ_SPECIALREG(EL1_REG(FAR))); + hypctx_write_sys_reg(hypctx, MAIR_EL1, + READ_SPECIALREG(EL1_REG(MAIR))); + hypctx_write_sys_reg(hypctx, SCTLR_EL1, + READ_SPECIALREG(EL1_REG(SCTLR))); + hypctx_write_sys_reg(hypctx, SPSR_EL1, + READ_SPECIALREG(EL1_REG(SPSR))); + hypctx_write_sys_reg(hypctx, TCR_EL1, + READ_SPECIALREG(EL1_REG(TCR))); /* TODO: Support when this is not res0 */ - hypctx->tcr2_el1 = 0; - hypctx->ttbr0_el1 = READ_SPECIALREG(EL1_REG(TTBR0)); - hypctx->ttbr1_el1 = READ_SPECIALREG(EL1_REG(TTBR1)); + hypctx_write_sys_reg(hypctx, TCR2_EL1, 0); + hypctx_write_sys_reg(hypctx, TTBR0_EL1, + READ_SPECIALREG(EL1_REG(TTBR0))); + hypctx_write_sys_reg(hypctx, TTBR1_EL1, + READ_SPECIALREG(EL1_REG(TTBR1))); } hypctx->cptr_el2 = READ_SPECIALREG(cptr_el2); @@ -320,30 +334,43 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest, WRITE_SPECIALREG(tpidrro_el0, hypctx->tpidrro_el0); WRITE_SPECIALREG(tpidr_el1, hypctx->tpidr_el1); - WRITE_SPECIALREG(actlr_el1, hypctx->actlr_el1); + WRITE_SPECIALREG(actlr_el1, hypctx_read_sys_reg(hypctx, ACTLR_EL1)); WRITE_SPECIALREG(csselr_el1, hypctx->csselr_el1); WRITE_SPECIALREG(mdccint_el1, hypctx->mdccint_el1); - WRITE_SPECIALREG(mdscr_el1, hypctx->mdscr_el1); + WRITE_SPECIALREG(mdscr_el1, hypctx_read_sys_reg(hypctx, MDSCR_EL1)); if (guest_or_nonvhe(guest)) { - WRITE_SPECIALREG(EL1_REG(ELR), hypctx->elr_el1); - WRITE_SPECIALREG(EL1_REG(VBAR), hypctx->vbar_el1); - - WRITE_SPECIALREG(EL1_REG(AFSR0), hypctx->afsr0_el1); - WRITE_SPECIALREG(EL1_REG(AFSR1), hypctx->afsr1_el1); - WRITE_SPECIALREG(EL1_REG(AMAIR), hypctx->amair_el1); - WRITE_SPECIALREG(EL1_REG(CONTEXTIDR), hypctx->contextidr_el1); - WRITE_SPECIALREG(EL1_REG(CPACR), hypctx->cpacr_el1); - WRITE_SPECIALREG(EL1_REG(ESR), hypctx->esr_el1); - WRITE_SPECIALREG(EL1_REG(FAR), hypctx->far_el1); - WRITE_SPECIALREG(EL1_REG(MAIR), hypctx->mair_el1); // - - WRITE_SPECIALREG(EL1_REG(SCTLR), hypctx->sctlr_el1); - WRITE_SPECIALREG(EL1_REG(SPSR), hypctx->spsr_el1); - WRITE_SPECIALREG(EL1_REG(TCR), hypctx->tcr_el1); + WRITE_SPECIALREG(EL1_REG(ELR), + hypctx_read_sys_reg(hypctx, ELR_EL1)); + WRITE_SPECIALREG(EL1_REG(VBAR), + hypctx_read_sys_reg(hypctx, VBAR_EL1)); + WRITE_SPECIALREG(EL1_REG(AFSR0), + hypctx_read_sys_reg(hypctx, AFSR0_EL1)); + WRITE_SPECIALREG(EL1_REG(AFSR1), + hypctx_read_sys_reg(hypctx, AFSR1_EL1)); + WRITE_SPECIALREG(EL1_REG(AMAIR), + hypctx_read_sys_reg(hypctx, AMAIR_EL1)); + WRITE_SPECIALREG(EL1_REG(CONTEXTIDR), + hypctx_read_sys_reg(hypctx, CONTEXTIDR_EL1)); + WRITE_SPECIALREG(EL1_REG(CPACR), + hypctx_read_sys_reg(hypctx, CPACR_EL1)); + WRITE_SPECIALREG(EL1_REG(ESR), + hypctx_read_sys_reg(hypctx, ESR_EL1)); + WRITE_SPECIALREG(EL1_REG(FAR), + hypctx_read_sys_reg(hypctx, FAR_EL1)); + WRITE_SPECIALREG(EL1_REG(MAIR), + hypctx_read_sys_reg(hypctx, MAIR_EL1)); + WRITE_SPECIALREG(EL1_REG(SCTLR), + hypctx_read_sys_reg(hypctx, SCTLR_EL1)); + WRITE_SPECIALREG(EL1_REG(SPSR), + hypctx_read_sys_reg(hypctx, SPSR_EL1)); + WRITE_SPECIALREG(EL1_REG(TCR), + hypctx_read_sys_reg(hypctx, TCR_EL1)); /* TODO: tcr2_el1 */ - WRITE_SPECIALREG(EL1_REG(TTBR0), hypctx->ttbr0_el1); - WRITE_SPECIALREG(EL1_REG(TTBR1), hypctx->ttbr1_el1); + WRITE_SPECIALREG(EL1_REG(TTBR0), + hypctx_read_sys_reg(hypctx, TTBR0_EL1)); + WRITE_SPECIALREG(EL1_REG(TTBR1), + hypctx_read_sys_reg(hypctx, TTBR1_EL1)); } if (guest) { @@ -542,7 +569,7 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest, #define LOAD_APR(x) \ case x: \ WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \ - hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \ + hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \ WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \ hypctx->vgic_v3_regs.ich_ap1r_el2[x]) LOAD_APR(3); @@ -568,6 +595,12 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx) uint64_t s1e1r, hpfar_el2; bool ecv_poff, hpfar_valid; + /* Allows uniform implementation of guest and host register reloads */ + host_hypctx.vncr_regs = hypctx->host_vncr_regs; +#ifndef VMM_VHE + host_hypctx.el2_vncr_addr = hypctx->el2_host_vncr_addr; +#endif + ecv_poff = (hyp->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0; vmm_hyp_reg_store(&host_hypctx, NULL, false, ecv_poff); #ifndef VMM_VHE diff --git a/sys/arm64/vmm/vmm_nvhe.c b/sys/arm64/vmm/vmm_nvhe.c index 025b1308ce68..3d42fc1d5900 100644 --- a/sys/arm64/vmm/vmm_nvhe.c +++ b/sys/arm64/vmm/vmm_nvhe.c @@ -36,6 +36,9 @@ #define EL1_REG(reg) MRS_REG_ALT_NAME(reg ## _EL1) #define EL0_REG(reg) MRS_REG_ALT_NAME(reg ## _EL0) +#define __hypctx_vncr_sysreg(hypctx, reg) \ + ((uint64_t*)(hypctx->el2_vncr_addr + REG_VNCR_OFFSET(reg))) + #include "vmm_hyp.c" uint64_t vmm_hyp_enter(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, diff --git a/sys/arm64/vmm/vmm_reset.c b/sys/arm64/vmm/vmm_reset.c index fd3414ecd066..4966de5c8fa5 100644 --- a/sys/arm64/vmm/vmm_reset.c +++ b/sys/arm64/vmm/vmm_reset.c @@ -55,19 +55,8 @@ reset_vm_el01_regs(void *vcpu) set_arch_unknown(el2ctx->tf); - set_arch_unknown(el2ctx->actlr_el1); - set_arch_unknown(el2ctx->afsr0_el1); - set_arch_unknown(el2ctx->afsr1_el1); - set_arch_unknown(el2ctx->amair_el1); - set_arch_unknown(el2ctx->contextidr_el1); - set_arch_unknown(el2ctx->cpacr_el1); set_arch_unknown(el2ctx->csselr_el1); - set_arch_unknown(el2ctx->elr_el1); - set_arch_unknown(el2ctx->esr_el1); - set_arch_unknown(el2ctx->far_el1); - set_arch_unknown(el2ctx->mair_el1); set_arch_unknown(el2ctx->mdccint_el1); - set_arch_unknown(el2ctx->mdscr_el1); set_arch_unknown(el2ctx->par_el1); /* @@ -77,19 +66,14 @@ reset_vm_el01_regs(void *vcpu) * SCTLR_CP15BEN: memory barrier instruction enable from EL0; RAO/WI * ~SCTLR_I: instruction cache off */ - el2ctx->sctlr_el1 = SCTLR_RES1; - el2ctx->sctlr_el1 &= ~SCTLR_M & ~SCTLR_C & ~SCTLR_I; - el2ctx->sctlr_el1 |= SCTLR_CP15BEN; + *hypctx_sys_reg(el2ctx, SCTLR_EL1) = SCTLR_RES1; + *hypctx_sys_reg(el2ctx, SCTLR_EL1) &= ~SCTLR_M & ~SCTLR_C & ~SCTLR_I; + *hypctx_sys_reg(el2ctx, SCTLR_EL1) |= SCTLR_CP15BEN; set_arch_unknown(el2ctx->sp_el0); - set_arch_unknown(el2ctx->tcr_el1); set_arch_unknown(el2ctx->tpidr_el0); set_arch_unknown(el2ctx->tpidr_el1); set_arch_unknown(el2ctx->tpidrro_el0); - set_arch_unknown(el2ctx->ttbr0_el1); - set_arch_unknown(el2ctx->ttbr1_el1); - set_arch_unknown(el2ctx->vbar_el1); - set_arch_unknown(el2ctx->spsr_el1); set_arch_unknown(el2ctx->dbgbcr_el1); set_arch_unknown(el2ctx->dbgbvr_el1);
