Hi to all,
I'm having a particular behavior on a new Ceph cluster. I've map
a RBD to a client and issue some performance tests with fio, at this
point everything goes just fine (also the results :) ), but then I try
to run another new test on a new RBD on the same client, and suddenly
the performance goes below 10MB/s and it took almost 10 minutes to
complete a 10G file test, if I issue a ceph -w I don't see anything
suspicious, any idea what can be happening here?
The cluster is made of:
3 x MON Servers
4 x OSD Servers (3TB SAS 6G disks for OSD daemons & tmpfs for Journal
-> there's one tmpfs of 36GB that is share by 9 OSD daemons, on each
server)
2 x Network SW (Cluster and Public)
10GbE speed on both networks
The ceph.conf file is the following:
[global]
fsid = 56e56e4c-ea59-4157-8b98-acae109bebe1
mon_initial_members = cephmon01, cephmon02, cephmon03
mon_host = 10.97.10.1,10.97.10.2,10.97.10.3
auth_client_required = cephx
auth_cluster_required = cephx
auth_service_required = cephx
filestore_xattr_use_omap = true
public_network = 10.97.0.0/16
cluster_network = 192.168.10.0/24
osd_pool_default_size = 2
glance_api_version = 2
[mon]
debug_optracker = 0
[mon.cephmon01]
host = cephmon01
mon_addr = 10.97.10.1:6789
[mon.cephmon02]
host = cephmon02
mon_addr = 10.97.10.2:6789
[mon.cephmon03]
host = cephmon03
mon_addr = 10.97.10.3:6789
[osd]
journal_dio = false
osd_journal_size = 4096
fstype = btrfs
debug_optracker = 0
[osd.0]
host = cephosd01
devs = /dev/sdc1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.1]
host = cephosd01
devs = /dev/sdd1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.2]
host = cephosd01
devs = /dev/sdf1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.3]
host = cephosd01
devs = /dev/sdg1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.4]
host = cephosd01
devs = /dev/sdi1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.5]
host = cephosd01
devs = /dev/sdj1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.6]
host = cephosd01
devs = /dev/sdl1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.7]
host = cephosd01
devs = /dev/sdm1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.8]
host = cephosd01
devs = /dev/sdn1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.9]
host = cephosd02
devs = /dev/sdc1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.10]
host = cephosd02
devs = /dev/sdd1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.11]
host = cephosd02
devs = /dev/sdf1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.12]
host = cephosd02
devs = /dev/sdg1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.13]
host = cephosd02
devs = /dev/sdi1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.14]
host = cephosd02
devs = /dev/sdj1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.15]
host = cephosd02
devs = /dev/sdl1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.16]
host = cephosd02
devs = /dev/sdm1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.17]
host = cephosd02
devs = /dev/sdn1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.18]
host = cephosd03
devs = /dev/sdc1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.19]
host = cephosd03
devs = /dev/sdd1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.20]
host = cephosd03
devs = /dev/sdf1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.21]
host = cephosd03
devs = /dev/sdg1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.22]
host = cephosd03
devs = /dev/sdi1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.23]
host = cephosd03
devs = /dev/sdj1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.24]
host = cephosd03
devs = /dev/sdl1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.25]
host = cephosd03
devs = /dev/sdm1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.26]
host = cephosd03
devs = /dev/sdn1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.27]
host = cephosd04
devs = /dev/sdc1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.28]
host = cephosd04
devs = /dev/sdd1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.29]
host = cephosd04
devs = /dev/sdf1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.30]
host = cephosd04
devs = /dev/sdg1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.31]
host = cephosd04
devs = /dev/sdi1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.32]
host = cephosd04
devs = /dev/sdj1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.33]
host = cephosd04
devs = /dev/sdl1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.34]
host = cephosd04
devs = /dev/sdm1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[osd.35]
host = cephosd04
devs = /dev/sdn1
osd_journal = /mnt/ramdisk/$cluster-$id-journal
[client.volumes]
keyring = /etc/ceph/ceph.client.volumes.keyring
Thanks in advance,
Best regards,
German Anders
_______________________________________________
ceph-users mailing list
[email protected]
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
