CPU stack traces from a hi1.4xlarge PV instance running ami-8baa73e2: CPU 0 is the only running CPU. The others are blocked.
CPU0 rip: ffffffff8105711a try_to_wake_up+0xca flags: 00001202 i nz rsp: ffff880f22dfc870 rax: 0000000000000008 rcx: 0000000000000000 rdx: 0000000000000002 rbx: ffff880012ca8000 rsi: ffff880e6af80048 rdi: 0000000000000001 rbp: ffff880f22dfc8c0 r8: 0000000000000000 r9: 0000000000000000 r10: 0000000000000000 r11: 0000000000000000 r12: ffff880012ca85f8 r13: 0000000000000000 r14: ffff880e6af80048 r15: 0000000000000003 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fd01a7e7700 gs: 0000 @ ffff880f22df9000/0000000000000000 Code (instr addr ffffffff8105711a) 66 90 45 85 ff 74 0e 44 8b 7d cc e9 a8 00 00 00 0f 1f 00 f3 90 <8b> 7b 28 85 ff 75 f7 48 8b 13 31 Stack: ffff880ec61086c0 ffff8801a1f5dde8 ffff880f22dfc890 0000000881079a45 0000000000000000 ffff880e6af81648 0000000000000001 ffff880ec6058708 0000000000000000 0000000000000000 ffff880f22dfc8d0 ffffffff81057262 ffff880f22dfc8f0 ffffffff810813c6 ffffea0015215a40 ffff880ec60586f0 Call Trace: [<ffffffff8105711a>] try_to_wake_up+0xca <-- [<ffffffff81057262>] default_wake_function+0x12 [<ffffffff810813c6>] autoremove_wake_function+0x16 [<ffffffff81044be8>] __wake_up_common+0x58 [<ffffffff81007aef>] xen_restore_fl_direct_reloc+0x4 [<ffffffff810466d8>] __wake_up+0x48 [<ffffffff812ceee6>] __freed_request+0x66 [<ffffffff812ceffa>] freed_request+0x3a [<ffffffff812cf083>] __blk_put_request.part.53+0x63 [<ffffffff812cf100>] __blk_put_request+0x50 [<ffffffff812cf1d1>] blk_finish_request+0xb1 [<ffffffff812cf26b>] __blk_end_request_all+0x4b [<ffffffff813e3677>] blkif_interrupt+0x167 [<ffffffff810ccb65>] handle_irq_event_percpu+0x55 [<ffffffff812ec66b>] radix_tree_lookup+0xb [<ffffffff810ccd7b>] handle_irq_event+0x4b [<ffffffff810cf53c>] handle_edge_irq+0x7c [<ffffffff81384a69>] __xen_evtchn_do_upcall+0x199 [<ffffffff813869ef>] xen_evtchn_do_upcall+0x2f [<ffffffff8160f62e>] xen_do_hypervisor_callback+0x1e [<ffffffff810013aa>] hypercall_page+0x3aa [<ffffffff810013aa>] hypercall_page+0x3aa [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e23>] xen_spin_lock_flags+0x63 [<ffffffff8160619e>] _raw_spin_lock_irqsave+0x2e [<ffffffff8104e662>] update_shares+0x92 [<ffffffff81056a68>] rebalance_domains+0x48 [<ffffffff81056d08>] run_rebalance_domains+0x48 [<ffffffff81065bf8>] __do_softirq+0xa8 [<ffffffff81384ad7>] __xen_evtchn_do_upcall+0x207 [<ffffffff8160f5dc>] call_softirq+0x1c [<ffffffff8100c265>] do_softirq+0x65 [<ffffffff81065fde>] irq_exit+0x8e [<ffffffff813869f5>] xen_evtchn_do_upcall+0x35 [<ffffffff8160f62e>] xen_do_hypervisor_callback+0x1e CPU1 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff88004e92dbf0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff88004e92dc08 rdi: 0000000000000003 rbp: ffff88004e92dc38 r8: ffff880ec85a3000 r9: ffff880f21c000d0 r10: 0000000000000001 r11: 0000000000000206 r12: 0000000000000017 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22e26900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f8f0c9cf700 gs: 0000 @ ffff880f22e14000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff88004e92dc24 ffffffff00000001 0000000000000000 0000000a81384b3e ffff88004e92dc38 ffff880f22e0b940 ffff88004e92dc48 ffffffff81386b80 ffff88004e92dc98 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff81604fbc>] do_nanosleep+0x9c [<ffffffff81085d08>] hrtimer_nanosleep+0xb8 [<ffffffff81084930>] update_rmtp+0x70 [<ffffffff81085724>] hrtimer_start_range_ns+0x14 [<ffffffff81085e27>] sys_nanosleep+0x57 [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU2 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880001ab7bf0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880001ab7c08 rdi: 0000000000000003 rbp: ffff880001ab7c38 r8: ffff880ec85a3600 r9: ffff880f21c00100 r10: 0000000000000001 r11: 0000000000000202 r12: 000000000000001d r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22e41900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f51a28f6700 gs: 0000 @ ffff880f22e2f000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880001ab7c24 ffffffff00000001 0000000000000000 0000001081384b3e ffff880001ab7c38 ffff880f22e0b940 ffff880001ab7c48 ffffffff81386b80 ffff880001ab7c98 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff81604fbc>] do_nanosleep+0x9c [<ffffffff81085d08>] hrtimer_nanosleep+0xb8 [<ffffffff81084930>] update_rmtp+0x70 [<ffffffff81085724>] hrtimer_start_range_ns+0x14 [<ffffffff81085e27>] sys_nanosleep+0x57 [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU3 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff8801a1ee7bf0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff8801a1ee7c08 rdi: 0000000000000003 rbp: ffff8801a1ee7c38 r8: ffff880ec85a3c00 r9: ffff880f21c00130 r10: 0000000000000001 r11: 0000000000000202 r12: 0000000000000023 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22e5c900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f9ee8860700 gs: 0000 @ ffff880f22e4a000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff8801a1ee7c24 ffffffff00000001 0000000000000000 0000001681384b3e ffff8801a1ee7c38 ffff880f22e0b940 ffff8801a1ee7c48 ffffffff81386b80 ffff8801a1ee7c98 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff81604fbc>] do_nanosleep+0x9c [<ffffffff81085d08>] hrtimer_nanosleep+0xb8 [<ffffffff81084930>] update_rmtp+0x70 [<ffffffff81085724>] hrtimer_start_range_ns+0x14 [<ffffffff81085e27>] sys_nanosleep+0x57 [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU4 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880ec7dd3680 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ec7dd3698 rdi: 0000000000000003 rbp: ffff880ec7dd36c8 r8: ffff880ec86e0200 r9: ffff880f21c00160 r10: 0000000000000001 r11: 0000000000000202 r12: 0000000000000029 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22e77900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f2f912ca700 gs: 0000 @ ffff880f22e65000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880ec7dd36b4 ffffffff00000001 0000000000000000 0000001c81384b3e ffff880ec7dd36c8 ffff880f22e0b940 ffff880ec7dd36d8 ffffffff81386b80 ffff880ec7dd3728 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff816051ac>] schedule_hrtimeout_range_clock+0x12c [<ffffffff81084930>] update_rmtp+0x70 [<ffffffff81085724>] hrtimer_start_range_ns+0x14 [<ffffffff81605203>] schedule_hrtimeout_range+0x13 [<ffffffff81179f89>] poll_schedule_timeout+0x49 [<ffffffff8117ab06>] do_select+0x4d6 [<ffffffff8117a090>] poll_freewait+0xe0 [<ffffffff8117a180>] __pollwait+0xf0 [<ffffffff81109022>] generic_perform_write+0x152 [<ffffffff8120b810>] ext4_dirty_inode+0x50 [<ffffffff811090ed>] generic_file_buffered_write+0x5d [<ffffffff8110aa59>] __generic_file_aio_write+0x229 [<ffffffff8160f60b>] xen_hypervisor_callback+0x1b [<ffffffff8160697a>] error_exit+0x2a [<ffffffff8160f60b>] xen_hypervisor_callback+0x1b [<ffffffff8160697a>] error_exit+0x2a [<ffffffff816064a1>] retint_restore_args+0x5 [<ffffffff8117adf1>] core_sys_select+0x1c1 [<ffffffff8160697a>] error_exit+0x2a [<ffffffff8160697a>] error_exit+0x2a [<ffffffff816064a1>] retint_restore_args+0x5 [<ffffffff81032b15>] pvclock_clocksource_read+0x55 [<ffffffff81032b15>] pvclock_clocksource_read+0x55 [<ffffffff810076e0>] xen_clocksource_read+0x20 [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff8108c0fd>] ktime_get_ts+0xad [<ffffffff8117b19b>] sys_select+0xbb [<ffffffff8100122a>] hypercall_page+0x22a [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU5 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880ec62c9680 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ec62c9698 rdi: 0000000000000003 rbp: ffff880ec62c96c8 r8: ffff880ec86e0800 r9: ffff880f21c00190 r10: 0000000000000001 r11: 0000000000000206 r12: 000000000000002f r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22e92900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f51acec9740 gs: 0000 @ ffff880f22e80000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880ec62c96b4 ffffffff00000001 0000000000000000 0000002281384b3e ffff880ec62c96c8 ffff880f22e0b940 ffff880ec62c96d8 ffffffff81386b80 ffff880ec62c9728 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff81032ad7>] pvclock_clocksource_read+0x17 [<ffffffff816040af>] schedule+0x3f [<ffffffff816051ac>] schedule_hrtimeout_range_clock+0x12c [<ffffffff81084930>] update_rmtp+0x70 [<ffffffff81085724>] hrtimer_start_range_ns+0x14 [<ffffffff81605203>] schedule_hrtimeout_range+0x13 [<ffffffff81179f89>] poll_schedule_timeout+0x49 [<ffffffff8117ab06>] do_select+0x4d6 [<ffffffff8117a090>] poll_freewait+0xe0 [<ffffffff8117a180>] __pollwait+0xf0 [<ffffffff8117a180>] __pollwait+0xf0 [<ffffffff81111b1f>] get_page_from_freelist+0x1cf [<ffffffff8110aa59>] __generic_file_aio_write+0x229 [<ffffffff81065ad7>] local_bh_enable_ip+0x27 [<ffffffff81112129>] __alloc_pages_nodemask+0x109 [<ffffffff81032b15>] pvclock_clocksource_read+0x55 [<ffffffff8160f60b>] xen_hypervisor_callback+0x1b [<ffffffff8160697a>] error_exit+0x2a [<ffffffff816064a1>] retint_restore_args+0x5 [<ffffffff8117adf1>] core_sys_select+0x1c1 [<ffffffff812ec66b>] radix_tree_lookup+0xb [<ffffffff81032b15>] pvclock_clocksource_read+0x55 [<ffffffff81032b15>] pvclock_clocksource_read+0x55 [<ffffffff810076e0>] xen_clocksource_read+0x20 [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff8108c0fd>] ktime_get_ts+0xad [<ffffffff8117b19b>] sys_select+0xbb [<ffffffff8160e6e3>] stub_clone+0x13 [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU6 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880046753bf0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880046753c08 rdi: 0000000000000003 rbp: ffff880046753c38 r8: ffff880ec86e0e00 r9: ffff880f21c001c0 r10: 0000000000000001 r11: 0000000000000206 r12: 0000000000000035 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22ead900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fdcaf5fc700 gs: 0000 @ ffff880f22e9b000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880046753c24 ffffffff00000001 0000000000000000 0000002881384b3e ffff880046753c38 ffff880f22e0b940 ffff880046753c48 ffffffff81386b80 ffff880046753c98 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff81604fbc>] do_nanosleep+0x9c [<ffffffff81085d08>] hrtimer_nanosleep+0xb8 [<ffffffff81084930>] update_rmtp+0x70 [<ffffffff81085724>] hrtimer_start_range_ns+0x14 [<ffffffff81085e27>] sys_nanosleep+0x57 [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU7 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880ec520b8a0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ec520b8b8 rdi: 0000000000000003 rbp: ffff880ec520b8e8 r8: ffff880ec86e1400 r9: ffff880f21c001f0 r10: 0000000039d98a10 r11: 0000000000000206 r12: 000000000000003b r13: 0000000000000001 r14: ffff880d5cec4601 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fab031bf700 gs: 0000 @ ffff880f22eb6000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000001 00000000fffffffa ffffffff8138512e ffff880ec520b8d4 ffffffff00000001 0000000000000000 0000002e81384b3e ffff880ec520b8e8 ffffffff81efce20 ffff880ec520b8f8 ffffffff81386b80 ffff880ec520b948 ffffffff815ecdcd 0000000000001000 0000000000000000 0000000000000001 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff814b4154>] _dm_request.isra.21+0x104 [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff81007b02>] check_events+0x12 [<ffffffff814b0260>] bitmap_unplug+0xe0 [<ffffffff814b0260>] bitmap_unplug+0xe0 [<ffffffff814b7c91>] stripe_merge+0x41 [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff8119a921>] __bio_add_page.part.15+0x101 [<ffffffff812ebe20>] prio_tree_next+0xb0 [<ffffffff81112aa7>] __writepage+0x17 [<ffffffff81113018>] write_cache_pages+0x208 [<ffffffff81112a90>] set_page_dirty_lock+0x50 [<ffffffff81007b02>] check_events+0x12 [<ffffffff811132ba>] generic_writepages+0x4a [<ffffffff81113e61>] do_writepages+0x21 [<ffffffff8110a61b>] __filemap_fdatawrite_range+0x5b [<ffffffff8110a672>] filemap_write_and_wait_range+0x52 [<ffffffff811936f2>] vfs_fsync_range+0x42 [<ffffffff811937bc>] vfs_fsync+0x1c [<ffffffff81137964>] sys_msync+0x144 [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU8 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880e6af81378 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880e6af81390 rdi: 0000000000000003 rbp: ffff880e6af813c0 r8: ffff880ec86e1a00 r9: ffff880f21c00490 r10: 0000000000000001 r11: 0000000000000202 r12: 0000000000000041 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22ee3900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f876e1e1700 gs: 0000 @ ffff880f22ed1000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880e6af813ac ffffffff00000001 0000000000000000 0000003481384b3e ffff880e6af813c0 ffff880f22e0b940 ffff880e6af813d0 ffffffff81386b80 ffff880e6af81420 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff8160415f>] io_schedule+0x8f [<ffffffff812cfa1d>] get_request_wait+0xcd [<ffffffff810813b0>] add_wait_queue+0x60 [<ffffffff812d0409>] __make_request+0x159 [<ffffffff814b4154>] _dm_request.isra.21+0x104 [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff81007b02>] check_events+0x12 [<ffffffff814b0260>] bitmap_unplug+0xe0 [<ffffffff814b0260>] bitmap_unplug+0xe0 [<ffffffff814b7c91>] stripe_merge+0x41 [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff8119a921>] __bio_add_page.part.15+0x101 [<ffffffff812ebe20>] prio_tree_next+0xb0 [<ffffffff81112aa7>] __writepage+0x17 [<ffffffff81113018>] write_cache_pages+0x208 [<ffffffff810096c9>] __switch_to+0xf9 [<ffffffff81112a90>] set_page_dirty_lock+0x50 [<ffffffff811132ba>] generic_writepages+0x4a [<ffffffff81113e61>] do_writepages+0x21 [<ffffffff8118efb3>] writeback_single_inode+0x103 [<ffffffff8118f3c1>] writeback_sb_inodes+0xe1 [<ffffffff8118f726>] writeback_inodes_wb+0xa6 [<ffffffff81007b02>] check_events+0x12 [<ffffffff8118fabf>] wb_writeback+0x34f [<ffffffff8118fc58>] wb_check_old_data_flush+0x98 [<ffffffff8118fdb1>] wb_do_writeback+0x151 [<ffffffff8106db30>] usleep_range+0x50 [<ffffffff8118fed3>] bdi_writeback_thread+0x83 [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff8118fe50>] wb_do_writeback+0x1f0 [<ffffffff8108090c>] kthread+0x8c [<ffffffff8160f4e4>] kernel_thread_helper+0x4 [<ffffffff8160e5e3>] int_ret_from_sys_call+0x7 [<ffffffff816064a1>] retint_restore_args+0x5 [<ffffffff8160f4e0>] gs_change+0x13 CPU9 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880078ea1bf0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880078ea1c08 rdi: 0000000000000003 rbp: ffff880078ea1c38 r8: ffff880ec8780000 r9: ffff880f21c004c0 r10: 0000000000000001 r11: 0000000000000206 r12: 0000000000000047 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22efe900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f2edebab700 gs: 0000 @ ffff880f22eec000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880078ea1c24 ffffffff00000001 0000000000000000 0000003a81384b3e ffff880078ea1c38 ffff880f22e0b940 ffff880078ea1c48 ffffffff81386b80 ffff880078ea1c98 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff81604fbc>] do_nanosleep+0x9c [<ffffffff81085d08>] hrtimer_nanosleep+0xb8 [<ffffffff81084930>] update_rmtp+0x70 [<ffffffff81085724>] hrtimer_start_range_ns+0x14 [<ffffffff81085e27>] sys_nanosleep+0x57 [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU10 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880ec50cdad8 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ec50cdaf0 rdi: 0000000000000003 rbp: ffff880ec50cdb20 r8: ffff880ec8780600 r9: ffff880f21c004f0 r10: 000000000e6664e8 r11: 0000000000000202 r12: 000000000000004d r13: 0000000000000001 r14: ffff880ec7feef01 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f9321a06700 gs: 0000 @ ffff880f22f07000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000001 00000000fffffffa ffffffff8138512e ffff880ec50cdb0c ffffffff00000001 0000000000000000 0000004081384b3e ffff880ec50cdb20 ffffffff81efce20 ffff880ec50cdb30 ffffffff81386b80 ffff880ec50cdb80 ffffffff815ecdcd 0000000000001000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff814b4154>] _dm_request.isra.21+0x104 [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff814b0260>] bitmap_unplug+0xe0 [<ffffffff814b0260>] bitmap_unplug+0xe0 [<ffffffff814b7c91>] stripe_merge+0x41 [<ffffffff814b3e41>] dm_merge_bvec+0xe1 [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff81603cb4>] __schedule+0x3d4 [<ffffffff8108090c>] kthread+0x8c [<ffffffff8160f4e4>] kernel_thread_helper+0x4 [<ffffffff8160e5e3>] int_ret_from_sys_call+0x7 [<ffffffff816064a1>] retint_restore_args+0x5 [<ffffffff8160f4e0>] gs_change+0x13 CPU11 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880158743bf0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880158743c08 rdi: 0000000000000003 rbp: ffff880158743c38 r8: ffff880ec8780c00 r9: ffff880f21c00520 r10: 0000000000000001 r11: 0000000000000202 r12: 0000000000000053 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22f34900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fd8820bd700 gs: 0000 @ ffff880f22f22000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880158743c24 ffffffff00000001 0000000000000000 0000004681384b3e ffff880158743c38 ffff880f22e0b940 ffff880158743c48 ffffffff81386b80 ffff880158743c98 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff81604fbc>] do_nanosleep+0x9c [<ffffffff81085d08>] hrtimer_nanosleep+0xb8 [<ffffffff81084930>] update_rmtp+0x70 [<ffffffff81085724>] hrtimer_start_range_ns+0x14 [<ffffffff81085e27>] sys_nanosleep+0x57 [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU12 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880ec7c037a0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ec7c037b8 rdi: 0000000000000003 rbp: ffff880ec7c037e8 r8: ffff880ec8781200 r9: ffff880f21c00550 r10: ffff880b02729c80 r11: 0000000000000202 r12: 0000000000000059 r13: 0000000000000001 r14: ffff880b02729c01 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fe87ad1b700 gs: 0000 @ ffff880f22f3d000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000200 00000000fffffffa ffffffff8138512e ffff880ec7c037d4 ffffffff00000001 0000000000000000 0000004c81384b3e ffff880ec7c037e8 ffffffff81efce20 ffff880ec7c037f8 ffffffff81386b80 ffff880ec7c03848 ffffffff815ecdcd 0000000000007000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff8115f0b7>] mem_cgroup_lru_add_after_commit+0x37 [<ffffffff8119aac3>] bio_add_page+0x53 [<ffffffff8119fd90>] do_mpage_readpage+0x470 [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff811a0010>] mpage_readpages+0x120 [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81112129>] __alloc_pages_nodemask+0x109 [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff81202ead>] ext4_readpages+0x1d [<ffffffff81114068>] read_pages+0x48 [<ffffffff81114283>] __do_page_cache_readahead+0x163 [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff811145f1>] ra_submit+0x21 [<ffffffff815f2a11>] do_sync_mmap_readahead.isra.26+0x94 [<ffffffff8110b53e>] filemap_fault+0x33e [<ffffffff8112b354>] __do_fault+0x54 [<ffffffff810097ba>] __switch_to+0x1ea [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff8112ea4a>] handle_pte_fault+0xfa [<ffffffff81005cce>] xen_pmd_val+0xe [<ffffffff81004759>] __raw_callee_save_xen_pmd_val+0x11 [<ffffffff8112ef18>] handle_mm_fault+0x1f8 [<ffffffff8105fb4c>] do_syslog+0x24c [<ffffffff81609a0e>] do_page_fault+0x14e [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff811679cd>] vfs_read+0x10d [<ffffffff81606715>] page_fault+0x25 CPU13 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001246 i z p rsp: ffff880ec8665ec8 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: ffff880ec8665fd8 rsi: 0000000000000000 rdi: 0000000000000001 rbp: ffff880ec8665ee0 r8: 0000000000000000 r9: 0000000000000000 r10: 0000000000000400 r11: 0000000000000246 r12: ffffffff81cc8640 r13: 000000000000000d r14: 0000000000000000 r15: 0000000000000000 cs: e033 ss: e02b ds: 002b es: 002b fs: 0000 @ 00007f9eed6a7700 gs: 0000 @ ffff880f22f58000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000ffffffff ffffffff81007320 ffff880ec8665f10 ffffffff81012133 ffff880ec8665fd8 ffffffff81cc8640 0000000000000000 0000000000000000 ffff880ec8665f40 ffffffff8100920b ffffffff81007aa9 65a4173d190f48a7 0000000000000000 0000000000000000 ffff880ec8665f50 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff81007320>] xen_safe_halt+0x10 [<ffffffff81012133>] default_idle+0x53 [<ffffffff8100920b>] cpu_idle+0xab [<ffffffff81007aa9>] xen_irq_enable_direct_reloc+0x4 [<ffffffff815e151c>] cpu_bringup_and_idle+0xe CPU14 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880ec55cd6a8 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ec55cd6c0 rdi: 0000000000000003 rbp: ffff880ec55cd6f0 r8: ffff880ec8781e00 r9: ffff880f21c005b0 r10: ffff880f22ffbe00 r11: 0000000000000202 r12: 0000000000000065 r13: 0000000000000001 r14: ffff880ec53ec701 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fdcb5e4f700 gs: 0000 @ ffff880f22f73000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000064 00000000fffffffa ffffffff8138512e ffff880ec55cd6dc ffffffff00000001 0000000000000000 0000005881384b3e ffff880ec55cd6f0 ffffffff81efce20 ffff880ec55cd700 ffffffff81386b80 ffff880ec55cd750 ffffffff815ecdcd 0000000000001000 0000000000000000 0000000000000001 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff8110bb33>] mempool_alloc+0x53 [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff8119b27a>] bio_alloc_bioset+0x5a [<ffffffff8119525b>] submit_bh+0xeb [<ffffffff811974c0>] __block_write_full_page+0x210 [<ffffffff81196da0>] end_buffer_async_read+0x140 [<ffffffff81207d20>] ext4_get_block_write+0x20 [<ffffffff81207d20>] ext4_get_block_write+0x20 [<ffffffff8119830b>] block_write_full_page_endio+0xdb [<ffffffff81196da0>] end_buffer_async_read+0x140 [<ffffffff81198355>] block_write_full_page+0x15 [<ffffffff81206827>] ext4_writepage+0xc7 [<ffffffff81112aa7>] __writepage+0x17 [<ffffffff81113018>] write_cache_pages+0x208 [<ffffffff81112a90>] set_page_dirty_lock+0x50 [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff81007aef>] xen_restore_fl_direct_reloc+0x4 [<ffffffff810043b7>] xen_mc_flush+0x97 [<ffffffff81007b02>] check_events+0x12 [<ffffffff811132ba>] generic_writepages+0x4a [<ffffffff812465be>] journal_submit_data_buffers+0xbe [<ffffffff81007b02>] check_events+0x12 [<ffffffff81246c87>] jbd2_journal_commit_transaction+0x2d7 [<ffffffff8100322e>] xen_end_context_switch+0x1e [<ffffffff81007aef>] xen_restore_fl_direct_reloc+0x4 [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff810813b0>] add_wait_queue+0x60 [<ffffffff8124ba6b>] kjournald2+0xbb [<ffffffff810813b0>] add_wait_queue+0x60 [<ffffffff8124b9b0>] commit_timeout+0x10 [<ffffffff8108090c>] kthread+0x8c [<ffffffff8160f4e4>] kernel_thread_helper+0x4 [<ffffffff8160e5e3>] int_ret_from_sys_call+0x7 [<ffffffff816064a1>] retint_restore_args+0x5 [<ffffffff8160f4e0>] gs_change+0x13 CPU15 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001246 i z p rsp: ffff880ec8671ec8 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: ffff880ec8671fd8 rsi: 0000000000000000 rdi: 0000000000000001 rbp: ffff880ec8671ee0 r8: 0000000000000000 r9: 0000000000000000 r10: ffff880ec8009c20 r11: 0000000000000246 r12: ffffffff81cc8640 r13: 000000000000000f r14: 0000000000000000 r15: 0000000000000000 cs: e033 ss: e02b ds: 002b es: 002b fs: 0000 @ 00007f9eeb991700 gs: 0000 @ ffff880f22f8e000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000001 00000000ffffffff ffffffff81007320 ffff880ec8671f10 ffffffff81012133 ffff880ec8671fd8 ffffffff81cc8640 0000000000000000 0000000000000000 ffff880ec8671f40 ffffffff8100920b ffffffff81007aa9 14997ba969c535f5 0000000000000000 0000000000000000 ffff880ec8671f50 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff81007320>] xen_safe_halt+0x10 [<ffffffff81012133>] default_idle+0x53 [<ffffffff8100920b>] cpu_idle+0xab [<ffffffff81007aa9>] xen_irq_enable_direct_reloc+0x4 [<ffffffff815e151c>] cpu_bringup_and_idle+0xe -- You received this bug notification because you are a member of Ubuntu Bugs, which is subscribed to Ubuntu. https://bugs.launchpad.net/bugs/1011792 Title: Scheduler deadlock running 3.0.0 on multiple EC2 instance types To manage notifications about this bug go to: https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1011792/+subscriptions -- ubuntu-bugs mailing list ubuntu-bugs@lists.ubuntu.com https://lists.ubuntu.com/mailman/listinfo/ubuntu-bugs
