One of our file servers internally to Sun that reproduces this running nv53
here is the dtrace output:
unix`mutex_vector_enter+0x120
zfs`metaslab_group_alloc+0x1a0
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_dva_allocate+0x54
zfs`zio_write_compress+0x248
zfs`arc_write+0xec
zfs`dbuf_sync+0x698
zfs`dnode_sync+0x2ec
zfs`dmu_objset_sync_dnodes+0x60
zfs`dmu_objset_sync+0x78
zfs`dsl_dataset_sync+0xc
zfs`dsl_pool_sync+0x64
zfs`spa_sync+0x1b4
zfs`txg_sync_thread+0x120
unix`thread_start+0x4
8
unix`mutex_vector_enter+0x120
zfs`metaslab_group_alloc+0x1a0
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_commit_writer+0x2ac
zfs`zil_commit+0x68
zfs`zfs_fsync+0xa4
genunix`fop_fsync+0x14
nfssrv`rfs3_setattr+0x4ec
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
8
genunix`avl_walk+0x3c
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_putpage+0x1d8
genunix`fop_putpage+0x1c
nfssrv`rfs3_commit+0x130
nfssrv`common_dispatch+0x4ec
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
8
genunix`avl_walk+0x40
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_commit_writer+0x2ac
zfs`zil_commit+0x68
zfs`zfs_putpage+0x1d8
genunix`fop_putpage+0x1c
nfssrv`rfs3_commit+0x130
nfssrv`common_dispatch+0x4ec
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
8
genunix`avl_walk+0x4c
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_dva_allocate+0x54
zfs`zio_write_compress+0x248
zfs`arc_write+0xec
zfs`dbuf_sync+0x698
zfs`dnode_sync+0x2ec
zfs`dmu_objset_sync_dnodes+0x60
zfs`dmu_objset_sync+0x50
zfs`dsl_dataset_sync+0xc
zfs`dsl_pool_sync+0x64
zfs`spa_sync+0x1b4
zfs`txg_sync_thread+0x120
unix`thread_start+0x4
8
zfs`fletcher_2_native+0x2c
zfs`arc_cksum_verify+0x64
zfs`arc_buf_thaw+0x38
zfs`dbuf_dirty+0x10c
zfs`dmu_write_uio+0xc4
zfs`zfs_write+0x3ac
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
8
zfs`fletcher_2_native+0x2c
zfs`arc_cksum_verify+0x64
zfs`arc_buf_destroy+0x1c
zfs`arc_evict+0x1f0
zfs`arc_adjust+0xf8
zfs`arc_kmem_reclaim+0x100
zfs`arc_kmem_reap_now+0x20
zfs`arc_reclaim_thread+0xdc
unix`thread_start+0x4
8
zfs`fletcher_2_native+0x2c
zfs`arc_cksum_compute+0x6c
zfs`dbuf_rele+0x40
zfs`dmu_buf_rele_array+0x34
zfs`dmu_write_uio+0x13c
zfs`zfs_write+0x3ac
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
8
unix`disp_getwork+0x7c
unix`idle+0xd0
unix`thread_start+0x4
9
unix`mutex_vector_enter+0x134
zfs`metaslab_free_dva+0xb0
zfs`metaslab_free+0x78
zfs`zio_dva_free+0x14
zfs`zio_wait+0xc
zfs`arc_free+0x13c
zfs`dsl_dataset_block_kill+0x168
zfs`dbuf_do_kill+0x14
genunix`taskq_thread+0x19c
unix`thread_start+0x4
9
genunix`avl_walk+0x3c
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_dva_allocate+0x54
zfs`zio_write_compress+0x248
zfs`arc_write+0xec
zfs`dbuf_sync+0x698
zfs`dnode_sync+0x2ec
zfs`dmu_objset_sync_dnodes+0x60
zfs`dmu_objset_sync+0x50
zfs`dsl_dataset_sync+0xc
zfs`dsl_pool_sync+0x64
zfs`spa_sync+0x1b4
zfs`txg_sync_thread+0x120
unix`thread_start+0x4
9
10
unix`mutex_vector_enter+0x120
zfs`metaslab_group_alloc+0x1a0
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_putpage+0x1d8
genunix`fop_putpage+0x1c
nfssrv`rfs3_commit+0x130
nfssrv`common_dispatch+0x4ec
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
10
unix`mutex_vector_enter+0x134
zfs`metaslab_group_alloc+0x1a0
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_commit_writer+0x2ac
zfs`zil_commit+0x68
zfs`zfs_fsync+0xa4
genunix`fop_fsync+0x14
nfssrv`rfs3_setattr+0x4ec
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
10
unix`disp_getwork+0x8c
unix`idle+0xd0
unix`thread_start+0x4
11
unix`mutex_vector_enter+0x134
zfs`metaslab_group_alloc+0x1a0
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_dva_allocate+0x54
zfs`zio_write_compress+0x248
zfs`arc_write+0xec
zfs`dbuf_sync+0x698
zfs`dnode_sync+0x2ec
zfs`dmu_objset_sync_dnodes+0x60
zfs`dmu_objset_sync+0x78
zfs`dsl_dataset_sync+0xc
zfs`dsl_pool_sync+0x64
zfs`spa_sync+0x1b4
zfs`txg_sync_thread+0x120
unix`thread_start+0x4
12
genunix`avl_walk+0x50
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_dva_allocate+0x54
zfs`zio_write_compress+0x248
zfs`arc_write+0xec
zfs`dbuf_sync+0x698
zfs`dnode_sync+0x2ec
zfs`dmu_objset_sync_dnodes+0x60
zfs`dmu_objset_sync+0x50
zfs`dsl_dataset_sync+0xc
zfs`dsl_pool_sync+0x64
zfs`spa_sync+0x1b4
zfs`txg_sync_thread+0x120
unix`thread_start+0x4
12
zfs`metaslab_ff_alloc+0x7c
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
12
genunix`avl_walk+0x40
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_dva_allocate+0x54
zfs`zio_write_compress+0x248
zfs`arc_write+0xec
zfs`dbuf_sync+0x698
zfs`dnode_sync+0x2ec
zfs`dmu_objset_sync_dnodes+0x60
zfs`dmu_objset_sync+0x50
zfs`dsl_dataset_sync+0xc
zfs`dsl_pool_sync+0x64
zfs`spa_sync+0x1b4
zfs`txg_sync_thread+0x120
unix`thread_start+0x4
13
unix`mutex_vector_enter+0x120
zfs`metaslab_free_dva+0xb0
zfs`metaslab_free+0x78
zfs`zio_dva_free+0x14
zfs`zio_wait+0xc
zfs`arc_free+0x13c
zfs`dsl_dataset_block_kill+0x168
zfs`dbuf_do_kill+0x14
genunix`taskq_thread+0x19c
unix`thread_start+0x4
18
genunix`avl_walk+0x50
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_commit_writer+0x2ac
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
18
zfs`metaslab_ff_alloc+0x74
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_commit_writer+0x2ac
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
19
genunix`avl_walk+0x4c
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_commit_writer+0x2ac
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
20
genunix`avl_walk+0x3c
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_commit_writer+0x2ac
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
22
unix`mutex_vector_enter+0x134
zfs`metaslab_group_alloc+0x1a0
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_putpage+0x1d8
genunix`fop_putpage+0x1c
nfssrv`rfs3_commit+0x130
nfssrv`common_dispatch+0x4ec
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
26
genunix`avl_walk+0x40
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_commit_writer+0x2ac
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
27
zfs`metaslab_ff_alloc+0x74
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
38
genunix`avl_walk+0x50
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
45
genunix`avl_walk+0x40
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
46
genunix`avl_walk+0x3c
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
57
genunix`avl_walk+0x4c
zfs`metaslab_ff_alloc+0x90
zfs`space_map_alloc+0x10
zfs`metaslab_group_alloc+0x200
zfs`metaslab_alloc_dva+0x10c
zfs`metaslab_alloc+0x3c
zfs`zio_alloc_blk+0x34
zfs`zil_lwb_write_start+0xdc
zfs`zil_lwb_commit+0x94
zfs`zil_commit_writer+0x1e4
zfs`zil_commit+0x68
zfs`zfs_write+0x648
genunix`fop_write+0x20
nfssrv`rfs3_write+0x3d8
nfssrv`common_dispatch+0x3c8
rpcmod`svc_getreq+0x20c
rpcmod`svc_run+0x1ac
nfs`nfssys+0x18c
unix`syscall_trap32+0xcc
57
This message posted from opensolaris.org
_______________________________________________
zfs-discuss mailing list
zfs-discuss@opensolaris.org
http://mail.opensolaris.org/mailman/listinfo/zfs-discuss
