The branch main has been updated by andrew: URL: https://cgit.FreeBSD.org/src/commit/?id=35164034e390f56f83efe5bf073d36812b19df23
commit 35164034e390f56f83efe5bf073d36812b19df23 Author: Kajetan Puchalski <[email protected]> AuthorDate: 2026-07-06 14:18:50 +0000 Commit: Andrew Turner <[email protected]> CommitDate: 2026-07-06 15:18:30 +0000 arm64/vmm: Make remaining registers use hypctx_*_sys_reg Move vgic, timer and trapframe registers into sys_regs to handle them in the same way as all the other registers. Signed-off-by: Kajetan Puchalski <[email protected]> Reviewed by: andrew Sponsored by: Arm Ltd Differential Revision: https://reviews.freebsd.org/D56556 --- sys/arm64/vmm/arm64.h | 46 +++++++++++-- sys/arm64/vmm/hyp_genassym.c | 35 ++++++++++ sys/arm64/vmm/io/vgic_v3.c | 46 +++++++------ sys/arm64/vmm/io/vgic_v3.h | 22 ------ sys/arm64/vmm/io/vtimer.c | 36 +++++----- sys/arm64/vmm/io/vtimer.h | 5 -- sys/arm64/vmm/vmm.c | 6 +- sys/arm64/vmm/vmm_arm64.c | 110 +++++++++++++++++------------- sys/arm64/vmm/vmm_hyp.c | 137 +++++++++++++++++++++----------------- sys/arm64/vmm/vmm_hyp_exception.S | 69 +++++++++---------- sys/arm64/vmm/vmm_reset.c | 6 +- sys/modules/vmm/Makefile | 11 +-- 12 files changed, 304 insertions(+), 225 deletions(-) diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h index 48ac3fe83ad7..5279ca546984 100644 --- a/sys/arm64/vmm/arm64.h +++ b/sys/arm64/vmm/arm64.h @@ -56,7 +56,14 @@ struct vgic_v3_cpu; #define DBGWCR_EL1(n) (DBGWCR0_EL1 + MIN(n, 15)) #define DBGWVR_EL1(n) (DBGWVR0_EL1 + MIN(n, 15)) +#define HOST_ICH_LR_EL2(n) (HOST_ICH_LR0_EL2 + MIN(n, VGIC_ICH_LR_NUM_MAX-1)) +#define HOST_ICH_AP0R_EL2(n) (HOST_ICH_AP0R0_EL2 + MIN(n, VGIC_ICH_APR_NUM_MAX-1)) +#define HOST_ICH_AP1R_EL2(n) (HOST_ICH_AP1R0_EL2 + MIN(n, VGIC_ICH_APR_NUM_MAX-1)) +#define GPR_X(n) (GPR_X0 + MIN(n, 30)) + enum hypctx_sysreg { + HOST_SPSR_EL2, + HOST_ESR_EL2, CSSELR_EL1, /* Cache Size Selection Register */ MDCCINT_EL1, /* Monitor DCC Interrupt Enable Register */ PAR_EL1, /* Physical Address Register */ @@ -120,6 +127,28 @@ enum hypctx_sysreg { HOST_FAR_EL2, /* Fault Address Register */ HOST_HPFAR_EL2, /* Hypervisor IPA Fault Address Register */ + HOST_ICH_EISR_EL2, /* End of Interrupt Status Register */ + HOST_ICH_ELRSR_EL2, /* Empty List Register Status Register */ + HOST_ICH_HCR_EL2, /* Hyp Control Register */ + HOST_ICH_MISR_EL2, /* Maintenance Interrupt State Register */ + HOST_ICH_VMCR_EL2, /* Virtual Machine Control Register */ + + /* + * The List Registers are part of the VM context and are modified on a + * world switch. They need to be allocated statically so they are + * mapped in the EL2 translation tables when struct hypctx is mapped. + */ + HOST_ICH_LR0_EL2, + HOST_ICH_LR_MAX_EL2 = HOST_ICH_LR0_EL2 + (VGIC_ICH_LR_NUM_MAX - 1), + /* Active Priorities Registers for Group 0 and 1 interrupts */ + HOST_ICH_AP0R0_EL2, + HOST_ICH_AP0R_MAX_EL2 = HOST_ICH_AP0R0_EL2 + (VGIC_ICH_APR_NUM_MAX - 1), + HOST_ICH_AP1R0_EL2, + HOST_ICH_AP1R_MAX_EL2 = HOST_ICH_AP1R0_EL2 + (VGIC_ICH_APR_NUM_MAX - 1), + + HOST_CNTHCTL_EL2, + HOST_CNTVOFF_EL2, + NR_NON_VNCR_REGS, /* VNCR Registers */ @@ -217,10 +246,17 @@ enum hypctx_sysreg { /* Virtual-address-sized registers */ VA_REGS_START, + /* Do not reorder these without matching asm changes */ + GPR_LR, + GPR_X0, + GPR_X30 = GPR_X0 + 30, + /* These can be reordered freely */ SP_EL0, /* Stack pointer */ TPIDR_EL0, /* EL0 Software ID Register */ TPIDRRO_EL0, /* Read-only Thread ID Register */ TPIDR_EL1, /* EL1 Software ID Register */ + HOST_SP_EL1, + HOST_ELR_EL2, VA_REGS_END, }; @@ -229,7 +265,6 @@ enum hypctx_sysreg { * Per-vCPU hypervisor state. */ struct hypctx { - struct trapframe tf; /* Virtual-address-sized registers */ uint64_t va_regs[VA_REGS_END - VA_REGS_START - 1]; /* Non-VNCR register state */ @@ -238,7 +273,6 @@ struct hypctx { struct hyp *hyp; struct vcpu *vcpu; - struct vtimer vtimer; struct vtimer_cpu vtimer_cpu; uint64_t setcaps; /* Currently enabled capabilities. */ @@ -247,7 +281,10 @@ struct hypctx { uint64_t debug_spsr; /* Saved guest SPSR */ uint64_t debug_mdscr; /* Saved guest MDSCR */ - struct vgic_v3_regs vgic_v3_regs; + struct { + uint16_t ich_lr_num; + uint16_t ich_apr_num; + } vgic_v3; struct vgic_v3_cpu *vgic_cpu; bool has_exception; bool dbg_oslock; @@ -277,6 +314,8 @@ struct hypctx { (&hypctx->va_regs[reg - VA_REGS_START - 1]) #endif +/* Calling this with markers like reg=VNCR_START or reg=NR_NON_VNCR_REGS + is a bad idea */ static inline uint64_t * hypctx_sys_reg(struct hypctx *hypctx, int reg /* enum hypctx_sysreg */) { @@ -285,7 +324,6 @@ hypctx_sys_reg(struct hypctx *hypctx, int reg /* enum hypctx_sysreg */) return (__hypctx_va_sysreg(hypctx, reg)); if (reg > VNCR_START) return (__hypctx_vncr_sysreg(hypctx, reg)); - /* Calling this with reg=VNCR_START or reg=NR_NON_VNCR_REGS is a bad idea */ return (&hypctx->sys_regs[reg]); } diff --git a/sys/arm64/vmm/hyp_genassym.c b/sys/arm64/vmm/hyp_genassym.c new file mode 100644 index 000000000000..da3e5f307ab2 --- /dev/null +++ b/sys/arm64/vmm/hyp_genassym.c @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/assym.h> +#include <sys/proc.h> + +#include "arm64.h" + +ASSYM(HYPCTX_GPR_LR, offsetof(struct hypctx, va_regs) + (GPR_LR - VA_REGS_START - 1) * sizeof(uint64_t)); +ASSYM(HYPCTX_GPR_X0, offsetof(struct hypctx, va_regs) + (GPR_X0 - VA_REGS_START - 1) * sizeof(uint64_t)); diff --git a/sys/arm64/vmm/io/vgic_v3.c b/sys/arm64/vmm/io/vgic_v3.c index f38fb4b74693..81bb94878368 100644 --- a/sys/arm64/vmm/io/vgic_v3.c +++ b/sys/arm64/vmm/io/vgic_v3.c @@ -503,7 +503,7 @@ vgic_v3_cpuinit(device_t dev, struct hypctx *hypctx) * * Maintenance interrupts are disabled. */ - hypctx->vgic_v3_regs.ich_hcr_el2 = ICH_HCR_EL2_En; + hypctx_write_sys_reg(hypctx, HOST_ICH_HCR_EL2, ICH_HCR_EL2_En); /* * Configure the Interrupt Controller Virtual Machine Control Register. @@ -518,20 +518,21 @@ vgic_v3_cpuinit(device_t dev, struct hypctx *hypctx) * ICH_VMCR_EL2_VENG0: virtual Group 0 interrupts enabled. * ICH_VMCR_EL2_VENG1: virtual Group 1 interrupts enabled. */ - hypctx->vgic_v3_regs.ich_vmcr_el2 = + hypctx_write_sys_reg(hypctx, HOST_ICH_VMCR_EL2, (virt_features.min_prio << ICH_VMCR_EL2_VPMR_SHIFT) | - ICH_VMCR_EL2_VBPR1_NO_PREEMPTION | ICH_VMCR_EL2_VBPR0_NO_PREEMPTION; - hypctx->vgic_v3_regs.ich_vmcr_el2 &= ~ICH_VMCR_EL2_VEOIM; - hypctx->vgic_v3_regs.ich_vmcr_el2 |= ICH_VMCR_EL2_VENG0 | + ICH_VMCR_EL2_VBPR1_NO_PREEMPTION | + ICH_VMCR_EL2_VBPR0_NO_PREEMPTION); + *hypctx_sys_reg(hypctx, HOST_ICH_VMCR_EL2) &= ~ICH_VMCR_EL2_VEOIM; + *hypctx_sys_reg(hypctx, HOST_ICH_VMCR_EL2) |= ICH_VMCR_EL2_VENG0 | ICH_VMCR_EL2_VENG1; - hypctx->vgic_v3_regs.ich_lr_num = virt_features.ich_lr_num; - for (i = 0; i < hypctx->vgic_v3_regs.ich_lr_num; i++) - hypctx->vgic_v3_regs.ich_lr_el2[i] = 0UL; + hypctx->vgic_v3.ich_lr_num = virt_features.ich_lr_num; + for (i = 0; i < hypctx->vgic_v3.ich_lr_num; i++) + hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i), 0UL); vgic_cpu->ich_lr_used = 0; TAILQ_INIT(&vgic_cpu->irq_act_pend); - hypctx->vgic_v3_regs.ich_apr_num = virt_features.ich_apr_num; + hypctx->vgic_v3.ich_apr_num = virt_features.ich_apr_num; } static void @@ -2118,7 +2119,7 @@ vgic_v3_flush_hwstate(device_t dev, struct hypctx *hypctx) */ mtx_lock_spin(&vgic_cpu->lr_mtx); - hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_UIE; + *hypctx_sys_reg(hypctx, HOST_ICH_HCR_EL2) &= ~ICH_HCR_EL2_UIE; /* Exit early if there are no buffered interrupts */ if (TAILQ_EMPTY(&vgic_cpu->irq_act_pend)) @@ -2128,33 +2129,34 @@ vgic_v3_flush_hwstate(device_t dev, struct hypctx *hypctx) __func__, vgic_cpu->ich_lr_used)); i = 0; - hypctx->vgic_v3_regs.ich_elrsr_el2 = - (1u << hypctx->vgic_v3_regs.ich_lr_num) - 1; + hypctx_write_sys_reg(hypctx, HOST_ICH_ELRSR_EL2, + (1u << hypctx->vgic_v3.ich_lr_num) - 1); TAILQ_FOREACH(irq, &vgic_cpu->irq_act_pend, act_pend_list) { /* No free list register, stop searching for IRQs */ - if (i == hypctx->vgic_v3_regs.ich_lr_num) + if (i == hypctx->vgic_v3.ich_lr_num) break; if (!irq->enabled) continue; - hypctx->vgic_v3_regs.ich_lr_el2[i] = ICH_LR_EL2_GROUP1 | - ((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) | - irq->irq; + hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i), + ICH_LR_EL2_GROUP1 | + ((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) | + irq->irq); if (irq->active) { - hypctx->vgic_v3_regs.ich_lr_el2[i] |= + *hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) |= ICH_LR_EL2_STATE_ACTIVE; } #ifdef notyet /* TODO: Check why this is needed */ if ((irq->config & _MASK) == LEVEL) - hypctx->vgic_v3_regs.ich_lr_el2[i] |= ICH_LR_EL2_EOI; + *hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) |= ICH_LR_EL2_EOI; #endif if (!irq->active && vgic_v3_irq_pending(irq)) { - hypctx->vgic_v3_regs.ich_lr_el2[i] |= + *hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) |= ICH_LR_EL2_STATE_PENDING; /* @@ -2196,8 +2198,8 @@ vgic_v3_sync_hwstate(device_t dev, struct hypctx *hypctx) * access unlocked. */ for (i = 0; i < vgic_cpu->ich_lr_used; i++) { - lr = hypctx->vgic_v3_regs.ich_lr_el2[i]; - hypctx->vgic_v3_regs.ich_lr_el2[i] = 0; + lr = hypctx_read_sys_reg(hypctx, HOST_ICH_LR_EL2(i)); + hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i), 0); irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), ICH_LR_EL2_VINTID(lr)); @@ -2244,7 +2246,7 @@ vgic_v3_sync_hwstate(device_t dev, struct hypctx *hypctx) vgic_v3_release_irq(irq); } - hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_EOICOUNT_MASK; + *hypctx_sys_reg(hypctx, HOST_ICH_HCR_EL2) &= ~ICH_HCR_EL2_EOICOUNT_MASK; vgic_cpu->ich_lr_used = 0; } diff --git a/sys/arm64/vmm/io/vgic_v3.h b/sys/arm64/vmm/io/vgic_v3.h index 8804cc7a0211..427a5ec5bb69 100644 --- a/sys/arm64/vmm/io/vgic_v3.h +++ b/sys/arm64/vmm/io/vgic_v3.h @@ -32,26 +32,4 @@ #define VGIC_ICH_LR_NUM_MAX 16 #define VGIC_ICH_APR_NUM_MAX 4 -/* Registers accessed by EL2 */ -struct vgic_v3_regs { - uint32_t ich_eisr_el2; /* End of Interrupt Status Register */ - uint32_t ich_elrsr_el2; /* Empty List register Status Register (ICH_ELRSR_EL2) */ - uint32_t ich_hcr_el2; /* Hyp Control Register */ - uint32_t ich_misr_el2; /* Maintenance Interrupt State Register */ - uint32_t ich_vmcr_el2; /* Virtual Machine Control Register */ - - /* - * The List Registers are part of the VM context and are modified on a - * world switch. They need to be allocated statically so they are - * mapped in the EL2 translation tables when struct hypctx is mapped. - */ - uint64_t ich_lr_el2[VGIC_ICH_LR_NUM_MAX]; - uint16_t ich_lr_num; - - /* Active Priorities Registers for Group 0 and 1 interrupts */ - uint16_t ich_apr_num; - uint32_t ich_ap0r_el2[VGIC_ICH_APR_NUM_MAX]; - uint32_t ich_ap1r_el2[VGIC_ICH_APR_NUM_MAX]; -}; - #endif /* !_VMM_VGIC_V3_H_ */ diff --git a/sys/arm64/vmm/io/vtimer.c b/sys/arm64/vmm/io/vtimer.c index ff29a06751e0..509e52947524 100644 --- a/sys/arm64/vmm/io/vtimer.c +++ b/sys/arm64/vmm/io/vtimer.c @@ -97,7 +97,7 @@ vtimer_virtual_timer_intr(void *arg) } cntpct_el0 = READ_SPECIALREG(cntpct_el0) - - hypctx->vtimer.cntvoff_el2; + hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2); if (hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 < cntpct_el0) vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), GT_VIRT_IRQ, true); @@ -186,17 +186,17 @@ vtimer_cpuinit(struct hypctx *hypctx) * * TODO: Don't trap when FEAT_ECV is present */ - hypctx->vtimer.cnthctl_el2 = + hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2, CNTHCTL_E2H_EL0VCTEN_NOTRAP | - CNTHCTL_E2H_EL0VTEN_NOTRAP; + CNTHCTL_E2H_EL0VTEN_NOTRAP); if (ecv_poff) { - hypctx->vtimer.cnthctl_el2 |= + *hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) |= CNTHCTL_E2H_EL0PCTEN_NOTRAP | CNTHCTL_E2H_EL0PTEN_NOTRAP | CNTHCTL_E2H_EL1PCTEN_NOTRAP | CNTHCTL_E2H_EL1PTEN_NOTRAP; } else { - hypctx->vtimer.cnthctl_el2 |= + *hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) |= CNTHCTL_E2H_EL0PCTEN_TRAP | CNTHCTL_E2H_EL0PTEN_TRAP | CNTHCTL_E2H_EL1PCTEN_TRAP | @@ -209,20 +209,20 @@ vtimer_cpuinit(struct hypctx *hypctx) * CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0 */ if (ecv_poff) { - hypctx->vtimer.cnthctl_el2 = + hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2, CNTHCTL_EL1PCTEN_NOTRAP | - CNTHCTL_EL1PCEN_NOTRAP; + CNTHCTL_EL1PCEN_NOTRAP); } else { - hypctx->vtimer.cnthctl_el2 = + hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2, CNTHCTL_EL1PCTEN_TRAP | - CNTHCTL_EL1PCEN_TRAP; + CNTHCTL_EL1PCEN_TRAP); } } if (ecv_poff) - hypctx->vtimer.cnthctl_el2 |= CNTHCTL_ECV_EN; + *hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) |= CNTHCTL_ECV_EN; - hypctx->vtimer.cntvoff_el2 = hypctx->hyp->cntvoff_el2; + hypctx_write_sys_reg(hypctx, HOST_CNTVOFF_EL2, hypctx->hyp->cntvoff_el2); } void @@ -280,10 +280,10 @@ vtimer_sync_hwstate(struct hypctx *hypctx) uint64_t cntpct_el0; cntpct_el0 = READ_SPECIALREG(cntpct_el0) - - hypctx->vtimer.cntvoff_el2; + hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2); vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.virt_timer, cntpct_el0); /* If FEAT_ECV_POFF is in use then we need to sync the physical timer */ - if ((hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0) { + if ((hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) & CNTHCTL_ECV_EN) != 0) { vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.phys_timer, cntpct_el0); } @@ -322,7 +322,7 @@ vtimer_schedule_irq(struct hypctx *hypctx, bool phys) else timer = &hypctx->vtimer_cpu.virt_timer; cntpct_el0 = READ_SPECIALREG(cntpct_el0) - - hypctx->vtimer.cntvoff_el2; + hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2); if (timer->cntx_cval_el0 < cntpct_el0) { /* Timer set in the past, trigger interrupt */ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), @@ -379,7 +379,7 @@ vtimer_phys_ctl_read(struct vcpu *vcpu, uint64_t *rval, void *arg) hypctx = vcpu_get_cookie(vcpu); vtimer_cpu = &hypctx->vtimer_cpu; - cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2; + cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2); if (vtimer_cpu->phys_timer.cntx_cval_el0 < cntpct_el0) /* Timer condition met */ *rval = vtimer_cpu->phys_timer.cntx_ctl_el0 | CNTP_CTL_ISTATUS; @@ -422,7 +422,7 @@ vtimer_phys_cnt_read(struct vcpu *vcpu, uint64_t *rval, void *arg) struct hypctx *hypctx; hypctx = vcpu_get_cookie(vcpu); - *rval = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2; + *rval = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2); return (0); } @@ -485,7 +485,7 @@ vtimer_phys_tval_read(struct vcpu *vcpu, uint64_t *rval, void *arg) *rval = (uint32_t)RES1; } else { cntpct_el0 = READ_SPECIALREG(cntpct_el0) - - hypctx->vtimer.cntvoff_el2; + hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2); *rval = vtimer_cpu->phys_timer.cntx_cval_el0 - cntpct_el0; } @@ -502,7 +502,7 @@ vtimer_phys_tval_write(struct vcpu *vcpu, uint64_t wval, void *arg) hypctx = vcpu_get_cookie(vcpu); vtimer_cpu = &hypctx->vtimer_cpu; - cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2; + cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2); vtimer_cpu->phys_timer.cntx_cval_el0 = (int32_t)wval + cntpct_el0; vtimer_remove_irq(hypctx, vcpu); diff --git a/sys/arm64/vmm/io/vtimer.h b/sys/arm64/vmm/io/vtimer.h index d415afa47acc..0fa772b8fcbf 100644 --- a/sys/arm64/vmm/io/vtimer.h +++ b/sys/arm64/vmm/io/vtimer.h @@ -37,11 +37,6 @@ struct hyp; struct hypctx; -struct vtimer { - uint64_t cnthctl_el2; - uint64_t cntvoff_el2; -}; - struct vtimer_timer { struct callout callout; struct mtx mtx; diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c index 07be8b1707a0..ad53dc16e91a 100644 --- a/sys/arm64/vmm/vmm.c +++ b/sys/arm64/vmm/vmm.c @@ -862,13 +862,13 @@ vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu) hypctx = vcpu_get_cookie(vcpu); - if ((hypctx->tf.tf_esr & ESR_ELx_ISS_MASK) != 0) + if ((hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_ISS_MASK) != 0) return (1); vme->exitcode = VM_EXITCODE_SMCCC; - vme->u.smccc_call.func_id = hypctx->tf.tf_x[0]; + vme->u.smccc_call.func_id = hypctx_read_sys_reg(hypctx, GPR_X(0)); for (i = 0; i < nitems(vme->u.smccc_call.args); i++) - vme->u.smccc_call.args[i] = hypctx->tf.tf_x[i + 1]; + vme->u.smccc_call.args[i] = hypctx_read_sys_reg(hypctx, GPR_X(i + 1)); *retu = true; return (0); diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c index ba826649092c..e2d667575706 100644 --- a/sys/arm64/vmm/vmm_arm64.c +++ b/sys/arm64/vmm/vmm_arm64.c @@ -659,7 +659,7 @@ arm64_gen_inst_emul_data(struct hypctx *hypctx, uint32_t esr_iss, ~(TTBR_ASID_MASK | TTBR_CnP); paging->tcr_el1 = hypctx_read_sys_reg(hypctx, TCR_EL1); paging->tcr2_el1 = hypctx_read_sys_reg(hypctx, TCR2_EL1); - paging->flags = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32); + paging->flags = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & (PSR_M_MASK | PSR_M_32); if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & SCTLR_M) != 0) paging->flags |= VM_GP_MMU_ENABLED; } @@ -685,7 +685,7 @@ raise_data_insn_abort(struct hypctx *hypctx, uint64_t far, bool dabort, int fsc) { uint64_t esr; - if ((hypctx->tf.tf_spsr & PSR_M_MASK) == PSR_M_EL0t) + if ((hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & PSR_M_MASK) == PSR_M_EL0t) esr = EXCP_INSN_ABORT_L << ESR_ELx_EC_SHIFT; else esr = EXCP_INSN_ABORT << ESR_ELx_EC_SHIFT; @@ -693,7 +693,7 @@ raise_data_insn_abort(struct hypctx *hypctx, uint64_t far, bool dabort, int fsc) if (dabort) esr |= EXCP_DATA_ABORT_L << ESR_ELx_EC_SHIFT; /* Set the IL bit if set by hardware */ - esr |= hypctx->tf.tf_esr & ESR_ELx_IL; + esr |= hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_IL; vmmops_exception(hypctx, esr | fsc, far); } @@ -705,8 +705,8 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret, uint64_t gpa; uint32_t esr_ec, esr_iss; - esr_ec = ESR_ELx_EXCEPTION(hypctx->tf.tf_esr); - esr_iss = hypctx->tf.tf_esr & ESR_ELx_ISS_MASK; + esr_ec = ESR_ELx_EXCEPTION(hypctx_read_sys_reg(hypctx, HOST_ESR_EL2)); + esr_iss = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_ISS_MASK; switch (esr_ec) { case EXCP_UNKNOWN: @@ -715,7 +715,7 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret, vme_ret->exitcode = VM_EXITCODE_HYP; break; case EXCP_TRAP_WFI_WFE: - if ((hypctx->tf.tf_esr & 0x3) == 0) { /* WFI */ + if ((hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & 0x3) == 0) { /* WFI */ vmm_stat_incr(hypctx->vcpu, VMEXIT_WFI, 1); vme_ret->exitcode = VM_EXITCODE_WFI; } else { @@ -744,7 +744,7 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret, case EXCP_DATA_ABORT_L: vmm_stat_incr(hypctx->vcpu, esr_ec == EXCP_DATA_ABORT_L ? VMEXIT_DATA_ABORT : VMEXIT_INSN_ABORT, 1); - switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) { + switch (hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ISS_DATA_DFSC_MASK) { case ISS_DATA_DFSC_TF_L0: case ISS_DATA_DFSC_TF_L1: case ISS_DATA_DFSC_TF_L2: @@ -769,7 +769,7 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret, if (vm_mem_allocated(hypctx->vcpu, gpa)) { vme_ret->exitcode = VM_EXITCODE_PAGING; vme_ret->inst_length = 0; - vme_ret->u.paging.esr = hypctx->tf.tf_esr; + vme_ret->u.paging.esr = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2); vme_ret->u.paging.gpa = gpa; } else if (esr_ec == EXCP_INSN_ABORT_L) { /* @@ -1094,7 +1094,7 @@ fault: int vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) { - uint64_t excp_type, vbar_el1; + uint64_t excp_type, vbar_el1, new_spsr; int handled; register_t daif; struct hyp *hyp; @@ -1108,36 +1108,43 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) vcpu = hypctx->vcpu; vme = vm_exitinfo(vcpu); - hypctx->tf.tf_elr = (uint64_t)pc; + hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, (uint64_t)pc); for (;;) { if (hypctx->has_exception) { hypctx->has_exception = false; - hypctx_write_sys_reg(hypctx, ELR_EL1, hypctx->tf.tf_elr); + hypctx_write_sys_reg(hypctx, ELR_EL1, + hypctx_read_sys_reg(hypctx, HOST_ELR_EL2)); - mode = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32); + mode = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & + (PSR_M_MASK | PSR_M_32); vbar_el1 = hypctx_read_sys_reg(hypctx, VBAR_EL1); if (mode == PSR_M_EL1t) { - hypctx->tf.tf_elr = vbar_el1 + 0x0; + hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, + vbar_el1 + 0x0); } else if (mode == PSR_M_EL1h) { - hypctx->tf.tf_elr = vbar_el1 + 0x200; + hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, + vbar_el1 + 0x200); } else if ((mode & PSR_M_32) == PSR_M_64) { /* 64-bit EL0 */ - hypctx->tf.tf_elr = vbar_el1 + 0x400; + hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, + vbar_el1 + 0x400); } else { /* 32-bit EL0 */ - hypctx->tf.tf_elr = vbar_el1 + 0x600; + hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, + vbar_el1 + 0x600); } /* Set the new spsr */ - hypctx_write_sys_reg(hypctx, SPSR_EL1, hypctx->tf.tf_spsr); + new_spsr = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2); + hypctx_write_sys_reg(hypctx, SPSR_EL1, new_spsr); /* Set the new cpsr */ - hypctx->tf.tf_spsr = hypctx_read_sys_reg(hypctx, - SPSR_EL1) & PSR_FLAGS; - - hypctx->tf.tf_spsr |= PSR_DAIF | PSR_M_EL1h; + hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2, + new_spsr & PSR_FLAGS); + *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) |= PSR_DAIF | + PSR_M_EL1h; /* * Update fields that may change on exeption entry @@ -1145,12 +1152,15 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) */ if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & SCTLR_SPAN) == 0) - hypctx->tf.tf_spsr |= PSR_PAN; + *hypctx_sys_reg(hypctx, + HOST_SPSR_EL2) |= PSR_PAN; if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & SCTLR_DSSBS) == 0) - hypctx->tf.tf_spsr &= ~PSR_SSBS; + *hypctx_sys_reg(hypctx, + HOST_SPSR_EL2) &= ~PSR_SSBS; else - hypctx->tf.tf_spsr |= PSR_SSBS; + *hypctx_sys_reg(hypctx, + HOST_SPSR_EL2) |= PSR_SSBS; } daif = intr_disable(); @@ -1170,7 +1180,8 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) /* Activate the stage2 pmap so the vmid is valid */ pmap_activate_vm(pmap); - hypctx_write_sys_reg(hypctx, HOST_VTTBR_EL2, pmap_to_ttbr0(pmap)); + hypctx_write_sys_reg(hypctx, HOST_VTTBR_EL2, + pmap_to_ttbr0(pmap)); /* * TODO: What happens if a timer interrupt is asserted exactly @@ -1195,10 +1206,10 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) if (excp_type == EXCP_TYPE_MAINT_IRQ) continue; - vme->pc = hypctx->tf.tf_elr; + vme->pc = hypctx_read_sys_reg(hypctx, HOST_ELR_EL2); vme->inst_length = INSN_SIZE; vme->u.hyp.exception_nr = excp_type; - vme->u.hyp.esr_el2 = hypctx->tf.tf_esr; + vme->u.hyp.esr_el2 = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2); vme->u.hyp.far_el2 = hypctx_read_sys_reg(hypctx, HOST_FAR_EL2); vme->u.hyp.hpfar_el2 = hypctx_read_sys_reg(hypctx, HOST_HPFAR_EL2); @@ -1209,7 +1220,7 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) break; else /* Resume guest execution from the next instruction. */ - hypctx->tf.tf_elr += vme->inst_length; + *hypctx_sys_reg(hypctx, HOST_ELR_EL2) += vme->inst_length; } return (0); @@ -1270,15 +1281,15 @@ hypctx_regptr(struct hypctx *hypctx, int reg) { switch (reg) { case VM_REG_GUEST_X0 ... VM_REG_GUEST_X29: - return (&hypctx->tf.tf_x[reg]); + return hypctx_sys_reg(hypctx, GPR_X(reg)); case VM_REG_GUEST_LR: - return (&hypctx->tf.tf_lr); + return hypctx_sys_reg(hypctx, GPR_LR); case VM_REG_GUEST_SP: - return (&hypctx->tf.tf_sp); + return hypctx_sys_reg(hypctx, HOST_SP_EL1); case VM_REG_GUEST_CPSR: - return (&hypctx->tf.tf_spsr); + return hypctx_sys_reg(hypctx, HOST_SPSR_EL2); case VM_REG_GUEST_PC: - return (&hypctx->tf.tf_elr); + return hypctx_sys_reg(hypctx, HOST_ELR_EL2); case VM_REG_GUEST_SCTLR_EL1: return hypctx_sys_reg(hypctx, SCTLR_EL1); case VM_REG_GUEST_TTBR0_EL1: @@ -1385,6 +1396,7 @@ vmmops_setcap(void *vcpui, int num, int val) { struct hypctx *hypctx = vcpui; int ret; + uint64_t host_spsr_el2; ret = 0; @@ -1402,38 +1414,42 @@ vmmops_setcap(void *vcpui, int num, int val) break; if (val != 0) { - hypctx->debug_spsr |= (hypctx->tf.tf_spsr & PSR_SS); - hypctx->debug_mdscr |= - (hypctx_read_sys_reg(hypctx, MDSCR_EL1) & MDSCR_SS); + hypctx->debug_spsr |= (hypctx_read_sys_reg(hypctx, + HOST_SPSR_EL2) & PSR_SS); + hypctx->debug_mdscr |= (hypctx_read_sys_reg(hypctx, + MDSCR_EL1) & MDSCR_SS); - hypctx->tf.tf_spsr |= PSR_SS; + *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) |= PSR_SS; *hypctx_sys_reg(hypctx, MDSCR_EL1) |= MDSCR_SS; *hypctx_sys_reg(hypctx, HOST_MDCR_EL2) |= MDCR_EL2_TDE; } else { - hypctx->tf.tf_spsr &= ~PSR_SS; - hypctx->tf.tf_spsr |= hypctx->debug_spsr; + *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) &= ~PSR_SS; + *hypctx_sys_reg(hypctx, + HOST_SPSR_EL2) |= hypctx->debug_spsr; hypctx->debug_spsr &= ~PSR_SS; *hypctx_sys_reg(hypctx, MDSCR_EL1) &= ~MDSCR_SS; - *hypctx_sys_reg(hypctx, MDSCR_EL1) |= hypctx->debug_mdscr; + *hypctx_sys_reg(hypctx, + MDSCR_EL1) |= hypctx->debug_mdscr; hypctx->debug_mdscr &= ~MDSCR_SS; if ((hypctx->setcaps & (1ul << VM_CAP_BRK_EXIT)) == 0) - *hypctx_sys_reg(hypctx, HOST_MDCR_EL2) &= ~MDCR_EL2_TDE; + *hypctx_sys_reg(hypctx, + HOST_MDCR_EL2) &= ~MDCR_EL2_TDE; } break; case VM_CAP_MASK_HWINTR: if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0)) break; + host_spsr_el2 = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2); if (val != 0) { - hypctx->debug_spsr |= (hypctx->tf.tf_spsr & - (PSR_I | PSR_F)); - hypctx->tf.tf_spsr |= PSR_I | PSR_F; + hypctx->debug_spsr |= (host_spsr_el2 & (PSR_I | PSR_F)); + host_spsr_el2 |= PSR_I | PSR_F; } else { - hypctx->tf.tf_spsr &= ~(PSR_I | PSR_F); - hypctx->tf.tf_spsr |= (hypctx->debug_spsr & - (PSR_I | PSR_F)); + host_spsr_el2 &= ~(PSR_I | PSR_F); + host_spsr_el2 |= (hypctx->debug_spsr & (PSR_I | PSR_F)); hypctx->debug_spsr &= ~(PSR_I | PSR_F); } + hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2, host_spsr_el2); break; default: ret = ENOENT; diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c index ffc51f243dcd..7a86086be56e 100644 --- a/sys/arm64/vmm/vmm_hyp.c +++ b/sys/arm64/vmm/vmm_hyp.c @@ -174,22 +174,27 @@ vmm_hyp_reg_store_pmu_debug(struct hypctx *hypctx, bool guest) static void vmm_hyp_reg_store_vgic(struct hypctx *hypctx, bool guest) { - hypctx->vgic_v3_regs.ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2); - hypctx->vgic_v3_regs.ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2); + hypctx_write_sys_reg(hypctx, HOST_ICH_HCR_EL2, + READ_SPECIALREG(ich_hcr_el2)); + hypctx_write_sys_reg(hypctx, HOST_ICH_VMCR_EL2, + READ_SPECIALREG(ich_vmcr_el2)); if (!guest) return; /* Store the GICv3 registers */ - hypctx->vgic_v3_regs.ich_eisr_el2 = READ_SPECIALREG(ich_eisr_el2); - hypctx->vgic_v3_regs.ich_elrsr_el2 = READ_SPECIALREG(ich_elrsr_el2); - hypctx->vgic_v3_regs.ich_misr_el2 = READ_SPECIALREG(ich_misr_el2); - - switch (hypctx->vgic_v3_regs.ich_lr_num - 1) { -#define STORE_LR(x) \ -case x: \ - hypctx->vgic_v3_regs.ich_lr_el2[x] = \ - READ_SPECIALREG(ich_lr ## x ##_el2) + hypctx_write_sys_reg(hypctx, HOST_ICH_EISR_EL2, + READ_SPECIALREG(ich_eisr_el2)); + hypctx_write_sys_reg(hypctx, HOST_ICH_ELRSR_EL2, + READ_SPECIALREG(ich_elrsr_el2)); + hypctx_write_sys_reg(hypctx, HOST_ICH_MISR_EL2, + READ_SPECIALREG(ich_misr_el2)); + + switch (hypctx->vgic_v3.ich_lr_num - 1) { +#define STORE_LR(x) \ +case x: \ + hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(x), \ + READ_SPECIALREG(ich_lr ## x ##_el2)) STORE_LR(15); STORE_LR(14); STORE_LR(13); @@ -210,13 +215,13 @@ case x: \ #undef STORE_LR } - switch (hypctx->vgic_v3_regs.ich_apr_num - 1) { -#define STORE_APR(x) \ -case x: \ - hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \ - READ_SPECIALREG(ich_ap0r ## x ##_el2); \ - hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \ - READ_SPECIALREG(ich_ap1r ## x ##_el2) + switch (hypctx->vgic_v3.ich_apr_num - 1) { +#define STORE_APR(x) \ +case x: \ + hypctx_write_sys_reg(hypctx, HOST_ICH_AP0R_EL2(x), \ + READ_SPECIALREG(ich_ap0r ## x ##_el2)); \ + hypctx_write_sys_reg(hypctx, HOST_ICH_AP1R_EL2(x), \ + READ_SPECIALREG(ich_ap1r ## x ##_el2)) STORE_APR(3); STORE_APR(2); STORE_APR(1); @@ -242,13 +247,14 @@ vmm_hyp_reg_store_timer(struct hypctx *hypctx, struct hyp *hyp, bool guest) } else { hypctx->vtimer_cpu.cntkctl_el1 = READ_SPECIALREG(cntkctl_el1); - hypctx->vtimer.cnthctl_el2 = - READ_SPECIALREG(cnthctl_el2); - hypctx->vtimer.cntvoff_el2 = - READ_SPECIALREG(cntvoff_el2); + hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2, + READ_SPECIALREG(cnthctl_el2)); + hypctx_write_sys_reg(hypctx, HOST_CNTVOFF_EL2, + READ_SPECIALREG(cntvoff_el2)); } - ecv_poff = (hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0; + ecv_poff = (hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) & + CNTHCTL_ECV_EN) != 0; if (guest_or_nonvhe(guest) && ecv_poff) { /* @@ -267,16 +273,17 @@ static void vmm_hyp_reg_store_special(struct hypctx *hypctx, struct hyp *hyp, bool guest) { /* Store the special to from the trapframe */ - hypctx->tf.tf_sp = READ_SPECIALREG(sp_el1); - hypctx->tf.tf_elr = READ_SPECIALREG(elr_el2); - hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2); + hypctx_write_sys_reg(hypctx, HOST_SP_EL1, READ_SPECIALREG(sp_el1)); + hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, READ_SPECIALREG(elr_el2)); + hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2, READ_SPECIALREG(spsr_el2)); if (guest) { - hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2); + hypctx_write_sys_reg(hypctx, HOST_ESR_EL2, + READ_SPECIALREG(esr_el2)); hypctx_write_sys_reg(hypctx, HOST_FAR_EL2, READ_SPECIALREG(far_el2)); - hypctx_write_sys_reg(hypctx, - PAR_EL1, READ_SPECIALREG(par_el1)); + hypctx_write_sys_reg(hypctx, PAR_EL1, + READ_SPECIALREG(par_el1)); } /* Store the guest special registers */ @@ -439,9 +446,9 @@ vmm_hyp_reg_restore_special(struct hypctx *hypctx, struct hyp *hyp, bool guest) hypctx_read_sys_reg(hypctx, HOST_VMPIDR_EL2)); /* Load the special regs from the trapframe */ - WRITE_SPECIALREG(sp_el1, hypctx->tf.tf_sp); - WRITE_SPECIALREG(elr_el2, hypctx->tf.tf_elr); - WRITE_SPECIALREG(spsr_el2, hypctx->tf.tf_spsr); + WRITE_SPECIALREG(sp_el1, hypctx_read_sys_reg(hypctx, HOST_SP_EL1)); + WRITE_SPECIALREG(elr_el2, hypctx_read_sys_reg(hypctx, HOST_ELR_EL2)); + WRITE_SPECIALREG(spsr_el2, hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2)); } static void @@ -581,10 +588,13 @@ static void vmm_hyp_reg_restore_timer(struct hypctx *hypctx, struct hyp *hyp, bool guest) { bool ecv_poff; - ecv_poff = (hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0; + ecv_poff = (hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) & + CNTHCTL_ECV_EN) != 0; - WRITE_SPECIALREG(cnthctl_el2, hypctx->vtimer.cnthctl_el2); - WRITE_SPECIALREG(cntvoff_el2, hypctx->vtimer.cntvoff_el2); + WRITE_SPECIALREG(cnthctl_el2, + hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2)); + WRITE_SPECIALREG(cntvoff_el2, + hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2)); if (guest) { WRITE_SPECIALREG(EL1_REG(CNTKCTL), @@ -600,7 +610,7 @@ vmm_hyp_reg_restore_timer(struct hypctx *hypctx, struct hyp *hyp, bool guest) * to keep in sync. */ WRITE_SPECIALREG(CNTPOFF_EL2_REG, - hypctx->vtimer.cntvoff_el2); + hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2)); isb(); } } else { @@ -623,18 +633,19 @@ static void vmm_hyp_reg_restore_vgic(struct hypctx *hypctx, bool guest) { /* Load the GICv3 registers */ - WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2); + WRITE_SPECIALREG(ich_hcr_el2, + hypctx_read_sys_reg(hypctx, HOST_ICH_HCR_EL2)); WRITE_SPECIALREG(ich_vmcr_el2, - hypctx->vgic_v3_regs.ich_vmcr_el2); + hypctx_read_sys_reg(hypctx, HOST_ICH_VMCR_EL2)); if (!guest) return; - switch (hypctx->vgic_v3_regs.ich_lr_num - 1) { -#define LOAD_LR(x) \ -case x: \ - WRITE_SPECIALREG(ich_lr ## x ##_el2, \ - hypctx->vgic_v3_regs.ich_lr_el2[x]) + switch (hypctx->vgic_v3.ich_lr_num - 1) { +#define LOAD_LR(x) \ +case x: \ + WRITE_SPECIALREG(ich_lr ## x ##_el2, \ + hypctx_read_sys_reg(hypctx, HOST_ICH_LR_EL2(x))) LOAD_LR(15); LOAD_LR(14); LOAD_LR(13); @@ -655,18 +666,18 @@ case x: \ #undef LOAD_LR } - switch (hypctx->vgic_v3_regs.ich_apr_num - 1) { -#define LOAD_APR(x) \ - case x: \ - WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \ - hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \ - WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \ - hypctx->vgic_v3_regs.ich_ap1r_el2[x]) - LOAD_APR(3); - LOAD_APR(2); - LOAD_APR(1); - default: - LOAD_APR(0); + switch (hypctx->vgic_v3.ich_apr_num - 1) { +#define LOAD_APR(x) \ +case x: \ + WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \ + hypctx_read_sys_reg(hypctx, HOST_ICH_AP0R_EL2(x))); \ + WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \ + hypctx_read_sys_reg(hypctx, HOST_ICH_AP1R_EL2(x))) + LOAD_APR(3); + LOAD_APR(2); + LOAD_APR(1); + default: + LOAD_APR(0); #undef LOAD_APR } } @@ -693,11 +704,12 @@ static void vmm_hyp_handle_guest_exit(struct hypctx *hypctx, uint64_t *ret) { bool hpfar_valid; - uint64_t s1e1r, hpfar_el2; + uint64_t s1e1r, hpfar_el2, host_esr_el2, host_far_el2; hpfar_valid = true; + host_esr_el2 = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2); if (*ret == EXCP_TYPE_EL1_SYNC) { - switch (ESR_ELx_EXCEPTION(hypctx->tf.tf_esr)) { + switch (ESR_ELx_EXCEPTION(host_esr_el2)) { case EXCP_INSN_ABORT_L: case EXCP_DATA_ABORT_L: /* @@ -713,9 +725,9 @@ vmm_hyp_handle_guest_exit(struct hypctx *hypctx, uint64_t *ret) * * TODO: Add a case for Arm erratum 834220. */ - if ((hypctx->tf.tf_esr & ISS_DATA_S1PTW) != 0) *** 186 LINES SKIPPED ***
