> Same host kernel?
yes, I also disabled KSM now, se below.
>
> (qemu) cpu 1
> (qemu) info registers
RAX=0000000000000000 RBX=0000000000000000 RCX=0000000000000002
RDX=000055a900000000
RSI=fffffa8003660450 RDI=0000000000000001 RBP=0000000000000080
RSP=fffff880009f7cc0
R8 =0000000000000000 R9 =0000000000000f44 R10=fffff8000145a000
R11=0000000000000000
R12=0000000000000000 R13=fffff800015cada0 R14=0000000000000000
R15=fffff880009bcec0
RIP=fffff80001575d5b RFL=00000202 [-------] CPL=0 II=0 A20=1 SMM=0 HLT=0
ES =002b 0000000000000000 ffffffff 00c0f300 DPL=3 DS [-WA]
CS =0010 0000000000000000 00000000 00209b00 DPL=0 CS64 [-RA]
SS =0018 0000000000000000 ffffffff 00c09300 DPL=0 DS [-WA]
DS =002b 0000000000000000 ffffffff 00c0f300 DPL=3 DS [-WA]
FS =0053 00000000fffe0000 00007c00 0040f300 DPL=3 DS [-WA]
GS =002b fffff880009b8000 ffffffff 00c0f300 DPL=3 DS [-WA]
LDT=0000 0000000000000000 ffffffff 00000000
TR =0040 fffff880009bcec0 00000067 00008b00 DPL=0 TSS64-busy
GDT= fffff880009c34c0 0000007f
IDT= fffff880009c3540 00000fff
CR0=80050031 CR2=fffff8800121ca30 CR3=0000000000187000 CR4=000006f8
DR0=0000000000000000 DR1=0000000000000000 DR2=0000000000000000
DR3=0000000000000000
DR6=00000000ffff0ff0 DR7=0000000000000400
EFER=0000000000000d01
FCW=027f FSW=3800 [ST=7] FTW=80 MXCSR=00001f80
FPR0=9fc0000000000000 4008 FPR1=0000000000000000 0000
FPR2=0000000000000000 0000 FPR3=0000000000000000 0000
FPR4=0000000000000000 0000 FPR5=0000000000000000 0000
FPR6=0000000000000000 0000 FPR7=0000000000000000 0000
XMM00=99b6438668a3a8ed8cfd2d2540e9389a XMM01=00000000000000000000000000000000
XMM02=00000000000000000000000000000000 XMM03=00000000000000000000000000000000
XMM04=00000000000000000000000000000000 XMM05=00000000000000000000000000000000
XMM06=00000000000000000000000000000000 XMM07=00000000000000000000000000000000
XMM08=00000000000000000000000000000000 XMM09=00000000000000000000000000000000
XMM10=00000000000000000000000000000000 XMM11=00000000000000000000000000000000
XMM12=00000000000000000000000000000000 XMM13=00000000000000000000000000000000
XMM14=00000000000000000000000000000000 XMM15=00000000000000000000000000000000
> (qemu) x/100i 0xfffff80001575d59 - 35
virsh # qemu-monitor-command 2 "x/100i 0xfffff80001575d59 - 35"
0xfffff80001575d36: mov %ebp,0x18(%rsp)
0xfffff80001575d3a: push %rsi
0xfffff80001575d3b: push %rdi
0xfffff80001575d3c: push %r12
0xfffff80001575d3e: sub $0x30,%rsp
0xfffff80001575d42: mov %cr8,%rbx
0xfffff80001575d46: mov $0x2,%ecx
0xfffff80001575d4b: cmp %cl,%bl
0xfffff80001575d4d: ja 0xfffff80001575d67
0xfffff80001575d4f: cmpq $0x0,0x1598d1(%rip) # 0xfffff800016cf628
0xfffff80001575d57: je 0xfffff80001575d5d
0xfffff80001575d59: pause
0xfffff80001575d5b: jmp 0xfffff80001575d4f
0xfffff80001575d5d: mov %cr8,%rax
0xfffff80001575d61: mov %rcx,%cr8
0xfffff80001575d65: mov %al,%bl
0xfffff80001575d67: mov %gs:0x20,%rdi
0xfffff80001575d70: xor %r9d,%r9d
0xfffff80001575d73: btl $0x10,0xe38c9(%rip) # 0xfffff80001659644
0xfffff80001575d7b: jae 0xfffff80001575d95
0xfffff80001575d7d: mov $0x1,%sil
0xfffff80001575d80: rdtsc
0xfffff80001575d82: mov 0x4700(%rdi),%r12d
0xfffff80001575d89: shl $0x20,%rdx
0xfffff80001575d8d: or %rdx,%rax
0xfffff80001575d90: mov %rax,%rbp
0xfffff80001575d93: jmp 0xfffff80001575da2
0xfffff80001575d95: mov 0x50(%rsp),%rbp
0xfffff80001575d9a: mov 0x50(%rsp),%r12d
0xfffff80001575d9f: xor %sil,%sil
0xfffff80001575da2: incl 0x4b00(%rdi)
0xfffff80001575da8: lock btsq $0x0,0x159876(%rip) # 0xfffff800016cf628
0xfffff80001575db2: jae 0xfffff80001575dd1
0xfffff80001575db4: lea 0x15986d(%rip),%rcx # 0xfffff800016cf628
0xfffff80001575dbb: callq 0xfffff8000148c1c0
0xfffff80001575dc0: incl 0x4b04(%rdi)
0xfffff80001575dc6: add %eax,0x4b08(%rdi)
0xfffff80001575dcc: mov %eax,%r9d
0xfffff80001575dcf: jmp 0xfffff80001575dd4
0xfffff80001575dd1: lfence
0xfffff80001575dd4: test %sil,%sil
0xfffff80001575dd7: je 0xfffff80001575e01
0xfffff80001575dd9: rdtsc
0xfffff80001575ddb: shl $0x20,%rdx
0xfffff80001575ddf: lea 0x159842(%rip),%rcx # 0xfffff800016cf628
0xfffff80001575de6: movb $0x0,0x28(%rsp)
0xfffff80001575deb: or %rdx,%rax
0xfffff80001575dee: mov %r12d,0x20(%rsp)
0xfffff80001575df3: mov %eax,%r8d
0xfffff80001575df6: mov %rax,%rdx
0xfffff80001575df9: sub %ebp,%r8d
0xfffff80001575dfc: callq 0xfffff80001560f10
0xfffff80001575e01: mov 0x60(%rsp),%rbp
0xfffff80001575e06: mov %bl,0x159818(%rip) # 0xfffff800016cf624
0xfffff80001575e0c: mov 0x58(%rsp),%rbx
0xfffff80001575e11: add $0x30,%rsp
0xfffff80001575e15: pop %r12
0xfffff80001575e17: pop %rdi
0xfffff80001575e18: pop %rsi
0xfffff80001575e19: retq
0xfffff80001575e1a: nop
0xfffff80001575e1b: nop
0xfffff80001575e1c: nop
0xfffff80001575e1d: nop
0xfffff80001575e1e: nop
0xfffff80001575e1f: nop
0xfffff80001575e20: sub $0x58,%rsp
0xfffff80001575e24: mov %r8b,0x40(%rsp)
0xfffff80001575e29: xor %eax,%eax
0xfffff80001575e2b: movb $0x1,0x38(%rsp)
0xfffff80001575e30: cmp %al,0x10(%rdx)
0xfffff80001575e33: mov %rax,0x30(%rsp)
0xfffff80001575e38: mov 0x14(%rdx),%eax
0xfffff80001575e3b: mov %eax,0x28(%rsp)
0xfffff80001575e3f: mov 0x20(%rdx),%rax
0xfffff80001575e43: lea 0x8(%rdx),%r9
0xfffff80001575e47: mov %rdx,%r8
0xfffff80001575e4a: mov 0x18(%rdx),%rdx
0xfffff80001575e4e: mov %rax,0x20(%rsp)
0xfffff80001575e53: je 0xfffff80001575e5c
0xfffff80001575e55: callq 0xfffff8000147b73c
0xfffff80001575e5a: jmp 0xfffff80001575e61
0xfffff80001575e5c: callq 0xfffff8000147b144
0xfffff80001575e61: add $0x58,%rsp
0xfffff80001575e65: retq
0xfffff80001575e66: nop
0xfffff80001575e67: nop
0xfffff80001575e68: nop
0xfffff80001575e69: nop
0xfffff80001575e6a: nop
0xfffff80001575e6b: nop
0xfffff80001575e6c: nop
0xfffff80001575e6d: nop
0xfffff80001575e6e: nop
0xfffff80001575e6f: nop
0xfffff80001575e70: mov %rbx,0x10(%rsp)
0xfffff80001575e75: mov %rbp,0x18(%rsp)
0xfffff80001575e7a: mov %rsi,0x20(%rsp)
0xfffff80001575e7f: push %rdi
0xfffff80001575e80: push %r12
> It's actually decelerated by synchronized_srcu_expedited(). It's one
> area which got a large slowdown as the price for the great scalability
> we achieved with srcu.
>
> But wait, this doesn't make sense. If we see mmio to vga, then bank
> switching is not involved, yet I see huge latencies on writes to vga io
> ports.
>
> Please install the qemu debuginfo package (if you built it yourself, I
> hope it was with debug symbols enabled) and run 'perf top'.
sure, here it is. it showed a lot of KSM activity:
2276.00 12.1% memcmp_pages [kernel.kallsyms]
1613.00 8.6% ksm_scan_thread [kernel.kallsyms]
hence I disabled KSM.
here's perf top then, checkdisk is still slow:
412.00 8.6% do_raw_spin_lock [kernel.kallsyms]
235.00 4.9% send_mono_rect /usr/bin/qemu-kvm
215.00 4.5% rb_next [kernel.kallsyms]
166.00 3.5% schedule [kernel.kallsyms]
141.00 3.0% add_preempt_count [kernel.kallsyms]
137.00 2.9% gen_rotc_rm_T1 /usr/bin/qemu-kvm
127.00 2.7% vmx_vcpu_run
/lib/modules/2.6.36lb.03/kernel/arch/x86/kvm/kvm-intel.ko
120.00 2.5% kvm_mmu_prepare_zap_page
/lib/modules/2.6.36lb.03/kernel/arch/x86/kvm/kvm.ko
103.00 2.2% sub_preempt_count [kernel.kallsyms]
102.00 2.1% debug_smp_processor_id [kernel.kallsyms]
87.00 1.8% delay_tsc [kernel.kallsyms]
62.00 1.3% cpupri_set [kernel.kallsyms]
60.00 1.3% vmcs_readl
/lib/modules/2.6.36lb.03/kernel/arch/x86/kvm/kvm-intel.ko
59.00 1.2% cpu_stopper_thread [kernel.kallsyms]
57.00 1.2% select_nohz_load_balancer [kernel.kallsyms]
56.00 1.2% memset /lib64/libc-2.5.so
52.00 1.1% alloc_vmap_area [kernel.kallsyms]
50.00 1.0% copy_user_generic_string [kernel.kallsyms]
46.00 1.0% gfn_to_memslot
/lib/modules/2.6.36lb.03/kernel/arch/x86/kvm/kvm.ko
38.00 0.8% is_shadow_present_pte
/lib/modules/2.6.36lb.03/kernel/arch/x86/kvm/kvm.ko
38.00 0.8% read_tsc [kernel.kallsyms]
35.00 0.7% sched_clock_local [kernel.kallsyms]
35.00 0.7% sched_clock [kernel.kallsyms]
35.00 0.7% cirrus_fill_src_or_dst_32 /usr/bin/qemu-kvm
33.00 0.7% cpu_stop_signal_done [kernel.kallsyms]
33.00 0.7% gen_rot_rm_T1 /usr/bin/qemu-kvm
31.00 0.7% do_raw_spin_unlock [kernel.kallsyms]
30.00 0.6% __enqueue_rt_entity [kernel.kallsyms]
30.00 0.6% gen_shift_rm_T1 /usr/bin/qemu-kvm
29.00 0.6% load_balance [kernel.kallsyms]
29.00 0.6% sched_clock_cpu [kernel.kallsyms]
29.00 0.6% _raw_spin_unlock_irqrestore [kernel.kallsyms]
28.00 0.6% __might_sleep [kernel.kallsyms]
28.00 0.6% finish_task_switch [kernel.kallsyms]
28.00 0.6% pick_next_task_rt [kernel.kallsyms]
28.00 0.6% tick_nohz_restart_sched_tick [kernel.kallsyms]
.
.
.
>
> I'm no longer sure this is the problem.
>
> --
> error compiling committee.c: too many arguments to function
>
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
--
-------------------------------------
Ing. Nikola CIPRICH
LinuxBox.cz, s.r.o.
28. rijna 168, 709 01 Ostrava
tel.: +420 596 603 142
fax: +420 596 621 273
mobil: +420 777 093 799
www.linuxbox.cz
mobil servis: +420 737 238 656
email servis: ser...@linuxbox.cz
-------------------------------------
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
