Hello,
i'm running a KVM Guest on SmartOS (build: 20160219T141818Z) and i try to sync
some files via rsync, then suddenly the KVM is shut down by the SmartOS host.
In the /zones/UUID/root/tmp/vm.log i'm getting the following error just before
the KVM is stopped:
###
Failed to allocate memory: Resource temporarily unavailable
###
The Core-Dump under /zones/UUID/cores shows me the following:
###
core 'core.qemu-system-x86_.27993' of 27993: /smartdc/bin/qemu-system-x86_64 -m
40000 -name 91ce03df-6ed9-648d-a018
----------------- lwp# 1 / thread# 1 --------------------
ffffdf7fff297ee4 __lwp_unpark () + 14
00000000004fc651 kvm_mutex_unlock () + 33
00000000004fc689 qemu_mutex_unlock_iothread () + 13
00000000004e083c main_loop_wait (0) + 267
00000000004fc316 kvm_main_loop () + 119
00000000004e0aca main_loop () + 17
00000000004e4818 main (30, ffffdf7fffdff8c8, ffffdf7fffdffa50) + 2da4
00000000004ceb5c _start () + 6c
----------------- lwp# 2 / thread# 2 --------------------
ffffdf7fff29e28a __sigtimedwait () + a
ffffdf7fff289437 sigwaitinfo (f61950, ffffdf7ffecfedf0) + 17
000000000050d119 sigwait_compat (f61950) + 59
ffffdf7fff297b5a _thrp_setup (ffffdf7fff070240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 3 / thread# 3 --------------------
ffffdf7fff29e9ca ioctl () + a
00000000004fb7c6 kvm_cpu_exec (fa0c60) + 1c
00000000004fbe2b kvm_main_loop_cpu (fa0c60) + 69
00000000004fbf3a ap_main_loop (fa0c60) + e9
ffffdf7fff297b5a _thrp_setup (ffffdf7fff070a40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 4 / thread# 4 --------------------
ffffdf7fff29e9ca ioctl () + a
00000000004fb7c6 kvm_cpu_exec (fbc510) + 1c
00000000004fbe2b kvm_main_loop_cpu (fbc510) + 69
00000000004fbf3a ap_main_loop (fbc510) + e9
ffffdf7fff297b5a _thrp_setup (ffffdf7fff071240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 5 / thread# 5 --------------------
ffffdf7fff29e9ca ioctl () + a
00000000004fb7c6 kvm_cpu_exec (fcbd80) + 1c
00000000004fbe2b kvm_main_loop_cpu (fcbd80) + 69
00000000004fbf3a ap_main_loop (fcbd80) + e9
ffffdf7fff297b5a _thrp_setup (ffffdf7fff071a40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 6 / thread# 6 --------------------
ffffdf7fff29e9ca ioctl () + a
00000000004fb7c6 kvm_cpu_exec (fdb5f0) + 1c
00000000004fbe2b kvm_main_loop_cpu (fdb5f0) + 69
00000000004fbf3a ap_main_loop (fdb5f0) + e9
ffffdf7fff297b5a _thrp_setup (ffffdf7fff072240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 7 / thread# 7 --------------------
ffffdf7fff29e9ca ioctl () + a
00000000004fb7c6 kvm_cpu_exec (feae60) + 1c
00000000004fbe2b kvm_main_loop_cpu (feae60) + 69
00000000004fbf3a ap_main_loop (feae60) + e9
ffffdf7fff297b5a _thrp_setup (ffffdf7fff072a40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 8 / thread# 8 --------------------
ffffdf7fff29e1aa __timer_settime () + a
000000000059ad98 multiticks_rearm_timer (743ca0) + 3cb
0000000000599384 qemu_rearm_alarm_timer (743ca0) + 31
0000000000599b2d qemu_mod_timer (9c5088f50, 2076737b24) + 129
00000000004edf9b virtio_net_handle_tx_timer (9c52634c0, 9c5270080) + ef
0000000000617daa virtio_queue_notify_vq (9c5270080) + 7f
0000000000617df1 virtio_queue_notify (9c52634c0, 1) + 45
00000000004f18ba virtio_ioport_write (9c5262c30, 10, 1) + 159
00000000004f1e13 virtio_pci_config_writew (9c5262c30, c050, 1) + 6e
000000000054c358 ioport_write (1, c050, 1) + 76
000000000054cb0e cpu_outw (c050, 1) + 3c
00000000004f8e1f kvm_handle_io (c050, ffffdf7fff048000, 1, 2, 1) + 10c
00000000004fa808 kvm_run (ffa6d0) + 29a
00000000004fb7c6 kvm_cpu_exec (ffa6d0) + 1c
00000000004fbe2b kvm_main_loop_cpu (ffa6d0) + 69
00000000004fbf3a ap_main_loop (ffa6d0) + e9
ffffdf7fff297b5a _thrp_setup (ffffdf7fff073240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 9 / thread# 9 --------------------
ffffdf7fff29e9ca ioctl () + a
00000000004fb7c6 kvm_cpu_exec (1009f40) + 1c
00000000004fbe2b kvm_main_loop_cpu (1009f40) + 69
00000000004fbf3a ap_main_loop (1009f40) + e9
ffffdf7fff297b5a _thrp_setup (ffffdf7fff073a40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 10 / thread# 10 --------------------
ffffdf7fff29e9ca ioctl () + a
00000000004fb7c6 kvm_cpu_exec (10197b0) + 1c
00000000004fbe2b kvm_main_loop_cpu (10197b0) + 69
00000000004fbf3a ap_main_loop (10197b0) + e9
ffffdf7fff297b5a _thrp_setup (ffffdf7fff074240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 11 / thread# 11 --------------------
ffffdf7fff29e28a __sigtimedwait () + a
ffffdf7fff289437 sigwaitinfo (9c51db550, ffffdf7ffd791df0) + 17
000000000050d119 sigwait_compat (9c51db550) + 59
ffffdf7fff297b5a _thrp_setup (ffffdf7fff074a40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 71 / thread# 71 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c50ca550, 9d29d2600) + 7a
000000000050c528 handle_aiocb_rw (9c50ca550) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff07a240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 72 / thread# 72 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c5042570, 9c57a1000) + 7a
000000000050c528 handle_aiocb_rw (9c5042570) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff075240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 73 / thread# 73 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c51a2010, 9cf688200) + 7a
000000000050c528 handle_aiocb_rw (9c51a2010) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff078a40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 74 / thread# 74 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c53c50d0, 9c531a600) + 7a
000000000050c528 handle_aiocb_rw (9c53c50d0) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff004a40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 75 / thread# 75 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c53c5030, 9d1a12200) + 7a
000000000050c528 handle_aiocb_rw (9c53c5030) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff076a40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 76 / thread# 76 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c5041410, 9cf936400) + 7a
000000000050c528 handle_aiocb_rw (9c5041410) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff07aa40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 77 / thread# 77 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c50ca4b0, 9d21f2400) + 7a
000000000050c528 handle_aiocb_rw (9c50ca4b0) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff078240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 78 / thread# 78 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c519db40, 9c51b2600) + 7a
000000000050c528 handle_aiocb_rw (9c519db40) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff07ca40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 79 / thread# 79 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c5042610, 9cfa67600) + 7a
000000000050c528 handle_aiocb_rw (9c5042610) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff07ba40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 80 / thread# 80 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c50414b0, 9c5373400) + 7a
000000000050c528 handle_aiocb_rw (9c50414b0) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff07d240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 81 / thread# 81 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c5042890, 9d2df3800) + 7a
000000000050c528 handle_aiocb_rw (9c5042890) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff002240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 82 / thread# 82 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c51b5750, 9d35d3a00) + 7a
000000000050c528 handle_aiocb_rw (9c51b5750) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff07fa40) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 83 / thread# 83 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c51b5430, 9d39f4c00) + 7a
000000000050c528 handle_aiocb_rw (9c51b5430) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff076240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 84 / thread# 84 --------------------
ffffdf7fff29ed7a __pwrite () + a
000000000050c30d handle_aiocb_rw_linear (9c51b56b0, 9d41d4e00) + 7a
000000000050c528 handle_aiocb_rw (9c51b56b0) + 165
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff000240) + 8a
ffffdf7fff297e70 _lwp_start ()
----------------- lwp# 85 / thread# 85 --------------------
ffffdf7fff29f0ea _lwp_kill () + a
ffffdf7fff2338f0 raise (6) + 20
ffffdf7fff20db78 abort () + 98
000000000054b172 qemu_oom_check (0) + 49
000000000054b1ab qemu_memalign (200, 7e0000) + 33
0000000000508a5d qemu_blockalign (f9dc70, 7e0000) + 4f
000000000050c485 handle_aiocb_rw (9c51b5570) + c2
000000000050c770 aio_thread (0) + 166
ffffdf7fff297b5a _thrp_setup (ffffdf7fff079240) + 8a
ffffdf7fff297e70 _lwp_start ()
###
KVM-Definition:
###
{
"zonename": "91ce03df-6ed9-648d-a018-f8c5ada2bd0d",
"autoboot": true,
"brand": "kvm",
"limit_priv":
"default,-file_link_any,-net_access,-proc_fork,-proc_info,-proc_session",
"v": 1,
"create_timestamp": "2016-03-27T11:07:12.103Z",
"cpu_shares": 100,
"max_lwps": 2000,
"max_msg_ids": 4096,
"max_sem_ids": 4096,
"max_shm_ids": 4096,
"max_shm_memory": 32256,
"zfs_io_priority": 100,
"billing_id": "00000000-0000-0000-0000-000000000000",
"owner_uuid": "00000000-0000-0000-0000-000000000000",
"hostname": "xxx",
"resolvers": [
"xxx",
"xxx"
],
"alias": "xxx",
"vcpus": 8,
"qemu_extra_opts": "-boot menu=on",
"disks": [
{
"path": "/dev/zvol/rdsk/zones/91ce03df-6ed9-648d-a018-f8c5ada2bd0d-disk0",
"boot": true,
"model": "virtio",
"media": "disk",
"image_size": 10240,
"image_uuid": "eb7ad0b2-6dcb-45bf-aa60-17f038c4b0a1",
"zfs_filesystem": "zones/91ce03df-6ed9-648d-a018-f8c5ada2bd0d-disk0",
"zpool": "zones",
"size": 234375,
"compression": "lz4",
"refreservation": 10240,
"block_size": 8192
}
],
"nics": [
{
"interface": "net0",
"mac": "82:83:90:bc:41:f4",
"vlan_id": xxx,
"nic_tag": "external",
"gateway": "xxx",
"gateways": [
"xxx"
],
"netmask": "xxx",
"ip": "xxx",
"ips": [
"xxx"
],
"model": "virtio",
"allow_ip_spoofing": true,
"primary": true
}
],
"max_physical_memory": 24256,
"max_locked_memory": 24256,
"max_swap": 24256,
"ram": 24000,
"uuid": "91ce03df-6ed9-648d-a018-f8c5ada2bd0d",
"zone_state": "running",
"zonepath": "/zones/91ce03df-6ed9-648d-a018-f8c5ada2bd0d",
"zoneid": 50,
"last_modified": "2016-04-05T14:32:58.000Z",
"firewall_enabled": false,
"server_uuid": "00000000-0000-0000-0000-0cc47aa9f3da",
"platform_buildstamp": "20160219T141818Z",
"state": "running",
"boot_timestamp": "2016-04-05T14:32:45.000Z",
"pid": 28880,
"customer_metadata": {},
"internal_metadata": {},
"routes": {},
"tags": {},
"quota": 10,
"zfs_root_recsize": 131072,
"zfs_filesystem": "zones/91ce03df-6ed9-648d-a018-f8c5ada2bd0d",
"zpool": "zones",
"snapshots": []
}
###
The rsync command syncs about 37.000 files with a size of 3.5GB.
The strange thing is: If i lower the RAM of the KVM to for example 24GB the
rsync runs without a problem. If i set the RAM to 32GB or 40GB then the KVM is
shut down. The Host is a 256GB Machine with more than enough free memory (about
30GB free, ARC is about 97GB).
Also when i throttle the rsync to about 2MB / sec it runs without a problem.
I'm able to reproduce this error on another host system with SmartOS
20160303T144736Z (KVM with 32GB -> shutdown, 24GB runs without a problem).
Has anybody had such a problem?
Thanks
Greets
Kilian
-------------------------------------------
smartos-discuss
Archives: https://www.listbox.com/member/archive/184463/=now
RSS Feed: https://www.listbox.com/member/archive/rss/184463/25769125-55cfbc00
Modify Your Subscription:
https://www.listbox.com/member/?member_id=25769125&id_secret=25769125-7688e9fb
Powered by Listbox: http://www.listbox.com
