Handle LRAT error exception with support for lrat mapping and invalidation.
Signed-off-by: Mihai Caraman <[email protected]> [[email protected]: addressed review feedback, refactoring, cleanup & other fixes] Signed-off-by: Laurentiu Tudor <[email protected]> --- arch/powerpc/include/asm/kvm_host.h | 1 + arch/powerpc/include/asm/kvm_ppc.h | 2 + arch/powerpc/include/asm/mmu-book3e.h | 12 ++++ arch/powerpc/include/asm/reg_booke.h | 14 +++++ arch/powerpc/kernel/asm-offsets.c | 1 + arch/powerpc/kvm/booke.c | 41 +++++++++++++ arch/powerpc/kvm/bookehv_interrupts.S | 9 ++- arch/powerpc/kvm/e500_mmu_host.c | 106 ++++++++++++++++++++++++++++++++++ arch/powerpc/kvm/e500mc.c | 4 ++ arch/powerpc/mm/fsl_booke_mmu.c | 10 ++++ 10 files changed, 199 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index e187b6a..b207a32 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -444,6 +444,7 @@ struct kvm_vcpu_arch { u32 eplc; u32 epsc; u32 oldpir; + u64 fault_lper; #endif #if defined(CONFIG_BOOKE) diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index c6ef05b..ac3574f 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -117,6 +117,8 @@ extern void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu); extern int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, enum xlate_instdata xlid, enum xlate_readwrite xlrw, struct kvmppc_pte *pte); +extern void kvmppc_lrat_map(struct kvm_vcpu *vcpu, gfn_t gfn); +extern void kvmppc_lrat_invalidate(struct kvm_vcpu *vcpu); extern struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id); diff --git a/arch/powerpc/include/asm/mmu-book3e.h b/arch/powerpc/include/asm/mmu-book3e.h index cd4f04a..9dbad3c 100644 --- a/arch/powerpc/include/asm/mmu-book3e.h +++ b/arch/powerpc/include/asm/mmu-book3e.h @@ -40,6 +40,8 @@ /* MAS registers bit definitions */ +#define MAS0_ATSEL 0x80000000 +#define MAS0_ATSEL_SHIFT 31 #define MAS0_TLBSEL_MASK 0x30000000 #define MAS0_TLBSEL_SHIFT 28 #define MAS0_TLBSEL(x) (((x) << MAS0_TLBSEL_SHIFT) & MAS0_TLBSEL_MASK) @@ -55,6 +57,7 @@ #define MAS0_WQ_CLR_RSRV 0x00002000 #define MAS1_VALID 0x80000000 +#define MAS1_VALID_SHIFT 31 #define MAS1_IPROT 0x40000000 #define MAS1_TID(x) (((x) << 16) & 0x3FFF0000) #define MAS1_IND 0x00002000 @@ -220,6 +223,12 @@ #define TLBILX_T_CLASS2 6 #define TLBILX_T_CLASS3 7 +/* LRATCFG bits */ +#define LRATCFG_ASSOC 0xFF000000 +#define LRATCFG_LASIZE 0x00FE0000 +#define LRATCFG_LPID 0x00002000 +#define LRATCFG_NENTRY 0x00000FFF + #ifndef __ASSEMBLY__ #include <asm/bug.h> @@ -297,6 +306,9 @@ struct tlb_core_data { /* For software way selection, as on Freescale TLB1 */ u8 esel_next, esel_max, esel_first; +#ifdef CONFIG_KVM_BOOKE_HV + u8 lrat_next, lrat_max; +#endif }; #ifdef CONFIG_PPC64 diff --git a/arch/powerpc/include/asm/reg_booke.h b/arch/powerpc/include/asm/reg_booke.h index 2fef74b..265f5ec 100644 --- a/arch/powerpc/include/asm/reg_booke.h +++ b/arch/powerpc/include/asm/reg_booke.h @@ -55,6 +55,8 @@ /* Special Purpose Registers (SPRNs)*/ #define SPRN_DECAR 0x036 /* Decrementer Auto Reload Register */ +#define SPRN_LPER 0x038 /* Logical Page Exception Register */ +#define SPRN_LPERU 0x039 /* Logical Page Exception Register Upper */ #define SPRN_IVPR 0x03F /* Interrupt Vector Prefix Register */ #define SPRN_USPRG0 0x100 /* User Special Purpose Register General 0 */ #define SPRN_SPRG3R 0x103 /* Special Purpose Register General 3 Read */ @@ -76,6 +78,7 @@ #define SPRN_DVC2 0x13F /* Data Value Compare Register 2 */ #define SPRN_LPID 0x152 /* Logical Partition ID */ #define SPRN_MAS8 0x155 /* MMU Assist Register 8 */ +#define SPRN_LRATCFG 0x156 /* LRAT Configuration Register */ #define SPRN_TLB0PS 0x158 /* TLB 0 Page Size Register */ #define SPRN_TLB1PS 0x159 /* TLB 1 Page Size Register */ #define SPRN_MAS5_MAS6 0x15c /* MMU Assist Register 5 || 6 */ @@ -371,6 +374,9 @@ #define ESR_ILK 0x00100000 /* Instr. Cache Locking */ #define ESR_PUO 0x00040000 /* Unimplemented Operation exception */ #define ESR_BO 0x00020000 /* Byte Ordering */ +#define ESR_DATA 0x00000400 /* Page Table Data Access */ +#define ESR_TLBI 0x00000200 /* Page Table TLB Ineligible */ +#define ESR_PT 0x00000100 /* Page Table Translation */ #define ESR_SPV 0x00000080 /* Signal Processing operation */ /* Bit definitions related to the DBCR0. */ @@ -669,6 +675,14 @@ #define EPC_EPID 0x00003fff #define EPC_EPID_SHIFT 0 +/* Bit definitions for LPER */ +#define LPER_ALPN 0x000FFFFFFFFFF000ULL +#define LPER_ALPN_SHIFT 12 +#define LPER_WIMGE 0x00000F80 +#define LPER_WIMGE_SHIFT 7 +#define LPER_LPS 0x0000000F +#define LPER_LPS_SHIFT 0 + /* * The IBM-403 is an even more odd special case, as it is much * older than the IBM-405 series. We put these down here incase someone diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index de62392..b73ceb0 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -745,6 +745,7 @@ int main(void) DEFINE(VCPU_HOST_MAS4, offsetof(struct kvm_vcpu, arch.host_mas4)); DEFINE(VCPU_HOST_MAS6, offsetof(struct kvm_vcpu, arch.host_mas6)); DEFINE(VCPU_EPLC, offsetof(struct kvm_vcpu, arch.eplc)); + DEFINE(VCPU_FAULT_LPER, offsetof(struct kvm_vcpu, arch.fault_lper)); #endif #ifdef CONFIG_KVM_EXIT_TIMING diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index ae458f0..2b9e5b1 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -1322,6 +1322,47 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, break; } +#if defined(PPC64) && defined(CONFIG_KVM_BOOKE_HV) + case BOOKE_INTERRUPT_LRAT_ERROR: + { + gfn_t gfn; + + /* + * Guest TLB management instructions (EPCR.DGTMI == 0) is not + * supported for now + */ + if (!(vcpu->arch.fault_esr & ESR_PT)) { + WARN_ONCE(1, "%s: Guest TLB management instructions not supported!\n", + __func__); + break; + } + + gfn = (vcpu->arch.fault_lper & LPER_ALPN) >> LPER_ALPN_SHIFT; + + idx = srcu_read_lock(&vcpu->kvm->srcu); + + if (kvm_is_visible_gfn(vcpu->kvm, gfn)) { + kvmppc_lrat_map(vcpu, gfn); + r = RESUME_GUEST; + } else if (vcpu->arch.fault_esr & ESR_DATA) { + vcpu->arch.paddr_accessed = (gfn << PAGE_SHIFT) + | (vcpu->arch.fault_dear & (PAGE_SIZE - 1)); + vcpu->arch.vaddr_accessed = + vcpu->arch.fault_dear; + + r = kvmppc_emulate_mmio(run, vcpu); + kvmppc_account_exit(vcpu, MMIO_EXITS); + } else { + kvmppc_booke_queue_irqprio(vcpu, + BOOKE_IRQPRIO_MACHINE_CHECK); + r = RESUME_GUEST; + } + + srcu_read_unlock(&vcpu->kvm->srcu, idx); + break; + } +#endif + case BOOKE_INTERRUPT_DEBUG: { r = kvmppc_handle_debug(run, vcpu); if (r == RESUME_HOST) diff --git a/arch/powerpc/kvm/bookehv_interrupts.S b/arch/powerpc/kvm/bookehv_interrupts.S index 81bd8a07..1e9fa2a 100644 --- a/arch/powerpc/kvm/bookehv_interrupts.S +++ b/arch/powerpc/kvm/bookehv_interrupts.S @@ -62,6 +62,7 @@ #define NEED_EMU 0x00000001 /* emulation -- save nv regs */ #define NEED_DEAR 0x00000002 /* save faulting DEAR */ #define NEED_ESR 0x00000004 /* save faulting ESR */ +#define NEED_LPER 0x00000008 /* save faulting LPER */ /* * On entry: @@ -159,6 +160,12 @@ PPC_STL r9, VCPU_FAULT_DEAR(r4) .endif + /* Only supported on 64-bit cores for now */ + .if \flags & NEED_LPER + mfspr r7, SPRN_LPER + std r7, VCPU_FAULT_LPER(r4) + .endif + b kvmppc_resume_host .endm @@ -279,7 +286,7 @@ kvm_handler BOOKE_INTERRUPT_DEBUG, EX_PARAMS(DBG), \ kvm_handler BOOKE_INTERRUPT_DEBUG, EX_PARAMS(CRIT), \ SPRN_CSRR0, SPRN_CSRR1, 0 kvm_handler BOOKE_INTERRUPT_LRAT_ERROR, EX_PARAMS(GEN), \ - SPRN_SRR0, SPRN_SRR1, (NEED_EMU | NEED_DEAR | NEED_ESR) + SPRN_SRR0, SPRN_SRR1, (NEED_EMU | NEED_DEAR | NEED_ESR | NEED_LPER) #else /* * For input register values, see arch/powerpc/include/asm/kvm_booke_hv_asm.h diff --git a/arch/powerpc/kvm/e500_mmu_host.c b/arch/powerpc/kvm/e500_mmu_host.c index 12d5c67..99ad88a 100644 --- a/arch/powerpc/kvm/e500_mmu_host.c +++ b/arch/powerpc/kvm/e500_mmu_host.c @@ -96,6 +96,112 @@ static inline void __write_host_tlbe(struct kvm_book3e_206_tlb_entry *stlbe, stlbe->mas2, stlbe->mas7_3); } +#if defined(CONFIG_64BIT) && defined(CONFIG_KVM_BOOKE_HV) +static int lrat_next(void) +{ + int next, this; + struct tlb_core_data *tcd = get_paca()->tcd_ptr; + + this = tcd->lrat_next; + next = this + 1; + if (unlikely(next >= tcd->lrat_max)) + next = 0; + tcd->lrat_next = next; + + return this; +} + +static void write_host_lrate(int tsize, gfn_t gfn, unsigned long pfn, + uint32_t lpid, bool valid) +{ + struct kvm_book3e_206_tlb_entry stlbe; + unsigned long flags; + + stlbe.mas1 = (valid ? MAS1_VALID : 0) | MAS1_TSIZE(tsize); + stlbe.mas2 = ((u64)gfn << PAGE_SHIFT); + stlbe.mas7_3 = ((u64)pfn << PAGE_SHIFT); + stlbe.mas8 = MAS8_TGS | lpid; + + local_irq_save(flags); + + __write_host_tlbe(&stlbe, MAS0_ATSEL | MAS0_ESEL(lrat_next()), lpid); + + local_irq_restore(flags); +} + +void kvmppc_lrat_map(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + struct kvm_memory_slot *slot; + unsigned long pfn; + unsigned long hva; + struct vm_area_struct *vma; + unsigned long psize; + int tsize; + unsigned long tsize_pages; + + slot = gfn_to_memslot(vcpu->kvm, gfn); + if (!slot) { + pr_err_ratelimited("%s: couldn't find memslot for gfn %lx!\n", + __func__, (long)gfn); + return; + } + + hva = slot->userspace_addr; + + down_read(¤t->mm->mmap_sem); + vma = find_vma(current->mm, hva); + if (vma && (hva >= vma->vm_start)) { + psize = vma_kernel_pagesize(vma); + } else { + pr_err_ratelimited("%s: couldn't find virtual memory address for gfn %lx!\n", + __func__, (long)gfn); + up_read(¤t->mm->mmap_sem); + return; + } + up_read(¤t->mm->mmap_sem); + + pfn = gfn_to_pfn_memslot(slot, gfn); + if (is_error_noslot_pfn(pfn)) { + pr_err_ratelimited("%s: couldn't get real page for gfn %lx!\n", + __func__, (long)gfn); + return; + } + + tsize = __ilog2(psize) - 10; + tsize_pages = 1 << (tsize + 10 - PAGE_SHIFT); + gfn &= ~(tsize_pages - 1); + pfn &= ~(tsize_pages - 1); + + write_host_lrate(tsize, gfn, pfn, vcpu->kvm->arch.lpid, true); + + kvm_release_pfn_clean(pfn); +} + +void kvmppc_lrat_invalidate(struct kvm_vcpu *vcpu) +{ + uint32_t mas0, mas1 = 0; + int esel; + unsigned long flags; + + local_irq_save(flags); + + /* LRAT does not have a dedicated instruction for invalidation */ + for (esel = 0; esel < get_paca()->tcd_ptr->lrat_max; esel++) { + mas0 = MAS0_ATSEL | MAS0_ESEL(esel); + mtspr(SPRN_MAS0, mas0); + asm volatile("isync; tlbre" : : : "memory"); + mas1 = mfspr(SPRN_MAS1) & ~MAS1_VALID; + mtspr(SPRN_MAS1, mas1); + asm volatile("isync; tlbwe" : : : "memory"); + } + /* Must clear mas8 for other host tlbwe's */ + mtspr(SPRN_MAS8, 0); + isync(); + + local_irq_restore(flags); +} +#endif /* CONFIG_64BIT && CONFIG_KVM_BOOKE_HV */ + /* * Acquire a mas0 with victim hint, as if we just took a TLB miss. * diff --git a/arch/powerpc/kvm/e500mc.c b/arch/powerpc/kvm/e500mc.c index cda695d..5856f8f 100644 --- a/arch/powerpc/kvm/e500mc.c +++ b/arch/powerpc/kvm/e500mc.c @@ -99,6 +99,10 @@ void kvmppc_e500_tlbil_all(struct kvmppc_vcpu_e500 *vcpu_e500) asm volatile("tlbilxlpid"); mtspr(SPRN_MAS5, 0); local_irq_restore(flags); + +#ifdef PPC64 + kvmppc_lrat_invalidate(&vcpu_e500->vcpu); +#endif } void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 pid) diff --git a/arch/powerpc/mm/fsl_booke_mmu.c b/arch/powerpc/mm/fsl_booke_mmu.c index 9c90e66..b0da4b9 100644 --- a/arch/powerpc/mm/fsl_booke_mmu.c +++ b/arch/powerpc/mm/fsl_booke_mmu.c @@ -194,6 +194,16 @@ static unsigned long map_mem_in_cams_addr(phys_addr_t phys, unsigned long virt, get_paca()->tcd.esel_next = i; get_paca()->tcd.esel_max = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY; get_paca()->tcd.esel_first = i; + +#ifdef CONFIG_KVM_BOOKE_HV + get_paca()->tcd.lrat_next = 0; + if (((mfspr(SPRN_MMUCFG) & MMUCFG_MAVN) == MMUCFG_MAVN_V2) && + (mfspr(SPRN_MMUCFG) & MMUCFG_LRAT)) { + get_paca()->tcd.lrat_max = mfspr(SPRN_LRATCFG) & LRATCFG_NENTRY; + } else { + get_paca()->tcd.lrat_max = 0; + } +#endif #endif return amount_mapped; -- 1.8.3.1 -- To unsubscribe from this list: send the line "unsubscribe kvm-ppc" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
