"rep ins" emulation is going through emulator now. This is slow because emulator knows how to write back only one datum at a time. This patch provides fast path for the instruction in certain conditions. The conditions are: DF flag is not set, destination memory is RAM and single datum does not cross page boundary. If fast path code fails it falls back to emulation.
Signed-off-by: Gleb Natapov <[email protected]> --- arch/x86/include/asm/kvm_host.h | 6 ++ arch/x86/kvm/svm.c | 20 +++++-- arch/x86/kvm/vmx.c | 25 +++++-- arch/x86/kvm/x86.c | 133 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 165 insertions(+), 19 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 0f52aae..e139bbc 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -404,6 +404,10 @@ struct kvm_vcpu_arch { /* emulate context */ struct x86_emulate_ctxt emulate_ctxt; + struct x86_fast_string_pio_ctxt { + unsigned long linear_addr; + u8 ad_bytes; + } fast_string_pio_ctxt; bool emulate_regs_need_sync_to_vcpu; bool emulate_regs_need_sync_from_vcpu; int (*complete_userspace_io)(struct kvm_vcpu *vcpu); @@ -764,6 +768,8 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); struct x86_emulate_ctxt; int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port); +int kvm_fast_string_pio_in(struct kvm_vcpu *vcpu, int size, unsigned short port, + u8 ad_bytes_idx); void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); int kvm_emulate_halt(struct kvm_vcpu *vcpu); int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 7a41878..f3e7bb3 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -1887,21 +1887,31 @@ static int io_interception(struct vcpu_svm *svm) { struct kvm_vcpu *vcpu = &svm->vcpu; u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ - int size, in, string; + int size, in, string, rep; unsigned port; ++svm->vcpu.stat.io_exits; string = (io_info & SVM_IOIO_STR_MASK) != 0; + rep = (io_info & SVM_IOIO_REP_MASK) != 0; in = (io_info & SVM_IOIO_TYPE_MASK) != 0; - if (string || in) - return emulate_instruction(vcpu, 0) == EMULATE_DONE; port = io_info >> 16; size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; svm->next_rip = svm->vmcb->control.exit_info_2; - skip_emulated_instruction(&svm->vcpu); - return kvm_fast_pio_out(vcpu, size, port); + if (!string && !in) { + skip_emulated_instruction(&svm->vcpu); + return kvm_fast_pio_out(vcpu, size, port); + } else if (string && in && rep) { + int addr_size = (io_info & SVM_IOIO_ASIZE_MASK) >> + SVM_IOIO_ASIZE_SHIFT; + int r = kvm_fast_string_pio_in(vcpu, size, port, + ffs(addr_size) - 1); + if (r != EMULATE_FAIL) + return r == EMULATE_DONE; + } + + return emulate_instruction(vcpu, 0) == EMULATE_DONE; } static int nmi_interception(struct vcpu_svm *svm) diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index eeeb4a2..dc06630 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -639,6 +639,7 @@ static unsigned long *vmx_msr_bitmap_longmode; static bool cpu_has_load_ia32_efer; static bool cpu_has_load_perf_global_ctrl; +static bool cpu_has_ins_outs_inst_info; static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); static DEFINE_SPINLOCK(vmx_vpid_lock); @@ -2522,6 +2523,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) if (((vmx_msr_high >> 18) & 15) != 6) return -EIO; + cpu_has_ins_outs_inst_info = vmx_msr_high & (1u << 22); + vmcs_conf->size = vmx_msr_high & 0x1fff; vmcs_conf->order = get_order(vmcs_config.size); vmcs_conf->revision_id = vmx_msr_low; @@ -4393,23 +4396,31 @@ static int handle_triple_fault(struct kvm_vcpu *vcpu) static int handle_io(struct kvm_vcpu *vcpu) { unsigned long exit_qualification; - int size, in, string; + int size, in, string, rep; unsigned port; exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - string = (exit_qualification & 16) != 0; in = (exit_qualification & 8) != 0; + string = (exit_qualification & 16) != 0; + rep = (exit_qualification & 32) != 0; ++vcpu->stat.io_exits; - if (string || in) - return emulate_instruction(vcpu, 0) == EMULATE_DONE; - port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; - skip_emulated_instruction(vcpu); - return kvm_fast_pio_out(vcpu, size, port); + if (!string && !in) { + skip_emulated_instruction(vcpu); + return kvm_fast_pio_out(vcpu, size, port); + } else if (string && in && rep && cpu_has_ins_outs_inst_info) { + u32 inst_info = vmcs_read32(VMX_INSTRUCTION_INFO); + int r = kvm_fast_string_pio_in(vcpu, size, port, + (inst_info >> 7) & 7); + if (r != EMULATE_FAIL) + return r == EMULATE_DONE; + } + + return emulate_instruction(vcpu, 0) == EMULATE_DONE; } static void diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5ce24a8..a245c4a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4485,10 +4485,22 @@ static void init_decode_cache(struct x86_emulate_ctxt *ctxt, ctxt->mem_read.end = 0; } +static enum x86emul_mode get_emulation_mode(struct kvm_vcpu *vcpu) +{ + int cs_db, cs_l; + + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + + return (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL : + (kvm_get_rflags(vcpu) & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 : + cs_l ? X86EMUL_MODE_PROT64 : + cs_db ? X86EMUL_MODE_PROT32 : + X86EMUL_MODE_PROT16; +} + static void init_emulate_ctxt(struct kvm_vcpu *vcpu) { struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; - int cs_db, cs_l; /* * TODO: fix emulate.c to use guest_read/write_register @@ -4498,15 +4510,10 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) */ cache_all_regs(vcpu); - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); ctxt->eflags = kvm_get_rflags(vcpu); ctxt->eip = kvm_rip_read(vcpu); - ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL : - (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 : - cs_l ? X86EMUL_MODE_PROT64 : - cs_db ? X86EMUL_MODE_PROT32 : - X86EMUL_MODE_PROT16; + ctxt->mode = get_emulation_mode(vcpu); ctxt->guest_mode = is_guest_mode(vcpu); init_decode_cache(ctxt, vcpu->arch.regs); @@ -4742,6 +4749,118 @@ int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port) } EXPORT_SYMBOL_GPL(kvm_fast_pio_out); +static int __kvm_fast_string_pio_in(struct kvm_vcpu *vcpu, int size, + unsigned short port, unsigned long addr, + int count) +{ + struct page *page; + gpa_t gpa; + char *kaddr; + int ret; + + gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL); + + if (gpa == UNMAPPED_GVA || + (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + return EMULATE_FAIL; + + page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); + if (is_error_page(page)) { + kvm_release_page_clean(page); + return EMULATE_FAIL; + } + + kaddr = kmap_atomic(page); + kaddr += offset_in_page(gpa); + + ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port, + kaddr, count); + + kunmap_atomic(kaddr); + if (ret) { + u8 ad_bytes = vcpu->arch.fast_string_pio_ctxt.ad_bytes; + unsigned long reg; + + reg = kvm_register_read(vcpu, VCPU_REGS_RCX); + kvm_register_address_increment(ad_bytes, ®, -count); + kvm_register_write(vcpu, VCPU_REGS_RCX, reg); + + reg = kvm_register_read(vcpu, VCPU_REGS_RDI); + kvm_register_address_increment(ad_bytes, ®, count * size); + kvm_register_write(vcpu, VCPU_REGS_RDI, reg); + + kvm_release_page_dirty(page); + return EMULATE_DONE; + } + kvm_release_page_clean(page); + return EMULATE_DO_MMIO; +} + +static int complete_fast_string_pio(struct kvm_vcpu *vcpu) +{ + unsigned long linear_addr = vcpu->arch.fast_string_pio_ctxt.linear_addr; + int r; + + BUG_ON(!vcpu->arch.pio.count); + + vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); + r = __kvm_fast_string_pio_in(vcpu, vcpu->arch.pio.size, + vcpu->arch.pio.port, linear_addr, vcpu->arch.pio.count); + BUG_ON(r == EMULATE_DO_MMIO); + if (r == EMULATE_FAIL) /* mem slot gone while we were not looking */ + vcpu->arch.pio.count = 0; /* drop the pio data */ + srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); + return 1; +} + +int kvm_fast_string_pio_in(struct kvm_vcpu *vcpu, int size, + unsigned short port, u8 ad_bytes_idx) +{ + unsigned long rdi = kvm_register_read(vcpu, VCPU_REGS_RDI); + unsigned long linear_addr = rdi + get_segment_base(vcpu, VCPU_SREG_ES); + unsigned long rcx = kvm_register_read(vcpu, VCPU_REGS_RCX), count; + u8 ad_bytes; + int r; + + if (rcx == 0) { + kvm_x86_ops->skip_emulated_instruction(vcpu); + return EMULATE_DONE; + } + if (kvm_get_rflags(vcpu) & X86_EFLAGS_DF) + return EMULATE_FAIL; + if (ad_bytes_idx > 2) + return EMULATE_FAIL; + + ad_bytes = (u8[]){2, 4, 8}[ad_bytes_idx]; + + rdi = kvm_address_mask(ad_bytes, rdi); + + count = (PAGE_SIZE - offset_in_page(rdi))/size; + + if (count == 0) /* 'in' crosses page boundry */ + return EMULATE_FAIL; + + count = min(count, kvm_address_mask(ad_bytes, rcx)); + + r = kvm_linearize_address(vcpu, get_emulation_mode(vcpu), + rdi, VCPU_SREG_ES, count, true, false, ad_bytes, + &linear_addr); + + if (r >= 0) + return EMULATE_FAIL; + + r = __kvm_fast_string_pio_in(vcpu, size, port, linear_addr, count); + + if (r != EMULATE_DO_MMIO) + return r; + + vcpu->arch.fast_string_pio_ctxt.linear_addr = linear_addr; + vcpu->arch.fast_string_pio_ctxt.ad_bytes = ad_bytes; + vcpu->arch.complete_userspace_io = complete_fast_string_pio; + return EMULATE_DO_MMIO; +} +EXPORT_SYMBOL_GPL(kvm_fast_string_pio_in); + static void tsc_bad(void *info) { __this_cpu_write(cpu_tsc_khz, 0); -- 1.7.7.3 -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
