Morning all,
We starting to apply load to our test cephfs system and are noticing some odd
latency numbers. We are using erasure coding for the cold data pools and
replication for our our cache tiers (not on ssd yet) . We noticed the following
high latency on one node and it seams to be slowing down writes and reads on
the cluster.
Our next step is break out mds, mgr, and mons to different machines but we
wanted to start the discussion here.
Here is a bunch of information you may find useful.
ceph.conf
[global]
fsid = XXXXX
mon_initial_members = ceph-mon1, ceph-mon2, ceph-mon3
mon_host = 10.141.167.238,10.141.160.251,10.141.161.249
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
cluster network = 10.85.8.0/22
public network = 10.141.0.0/16
# we tested this with bluestore and xfs and have the same results
[osd]
enable_experimental_unrecoverable_data_corrupting_features = bluestore
Status
cluster 8f6ba9d6-314d-4725-bcfa-340e500697f0
health HEALTH_OK
monmap e2: 3 mons at
{ceph-mon1=10.141.167.238:6789/0,ceph-mon2=10.141.160.251:6789/0,ceph-mon3=10.141.161.249:6789/0}
election epoch 12, quorum 0,1,2 ceph-mon2,ceph-mon3,ceph-mon1
fsmap e30: 1/1/1 up {0=ceph-mon3=up:active}, 2 up:standby
mgr active: ceph-mon3 standbys: ceph-mon2, ceph-mon1
osdmap e100: 12 osds: 12 up, 12 in
flags sortbitwise,require_jewel_osds,require_kraken_osds
pgmap v119525: 124 pgs, 6 pools, 471 GB data, 1141 kobjects
970 GB used, 2231 GB / 3202 GB avail
124 active+clean
client io 11962 B/s rd, 11 op/s rd, 0 op/s wr
Pool space usage
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
3202G 2231G 970G 30.31
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
rbd 0 0 0 580G 0
cephfs-hot 1 76137M 11.35 580G 466451
cephfs-cold 2 397G 25.48 1161G 650158
cephfs_metadata 3 47237k 0 580G 52275
one-hot 4 0 0 580G 0
one 5 0 0 1161G 0
OSD Performance and Latency
osd commit_latency(ms) apply_latency(ms)
9 1 1
8 1 1
0 13 13
11 1 1
1 38 38
10 2 2
2 21 21
3 2 2
4 20 20
5 1 1
6 1 1
7 1 1
OSD Tree and Status
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 3.12685 root default
-2 1.08875 host ceph-mon1
0 0.27219 osd.0 up 1.00000 1.00000
1 0.27219 osd.1 up 1.00000 1.00000
2 0.27219 osd.2 up 1.00000 1.00000
4 0.27219 osd.4 up 1.00000 1.00000
-3 0.94936 host ceph-mon2
3 0.27219 osd.3 up 1.00000 1.00000
5 0.27219 osd.5 up 1.00000 1.00000
7 0.27219 osd.7 up 1.00000 1.00000
9 0.13280 osd.9 up 1.00000 1.00000
-4 1.08875 host ceph-mon3
6 0.27219 osd.6 up 1.00000 1.00000
8 0.27219 osd.8 up 1.00000 1.00000
10 0.27219 osd.10 up 1.00000 1.00000
11 0.27219 osd.11 up 1.00000 1.00000
[root@ceph-mon1 ~]# ceph pg dump
dumped all in format plain
version 119550
stamp 2017-03-15 12:46:24.428486
last_osdmap_epoch 100
last_pg_scan 100
full_ratio 0.95
nearfull_ratio 0.85
PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG
DISK_LOG STATE STATE_STAMP VERSION REPORTED UP
UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP
LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
0.2d 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.119959 0'0 99:57 [9,11,1]
9 [9,11,1] 9 0'0 2017-03-15 08:40:09.815059
0'0 2017-03-14 08:12:37.425834
0.2c 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:27:47.959479 0'0 100:29 [8,0,3]
8 [8,0,3] 8 0'0 2017-03-15 11:27:47.959398
0'0 2017-03-12 22:50:44.393457
0.2b 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:45.798794 0'0 94:27 [9,6,0]
9 [9,6,0] 9 0'0 2017-03-14 05:48:38.301001
0'0 2017-03-12 22:50:44.393456
0.2a 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:44.783652 0'0 93:26 [11,7,0]
11 [11,7,0] 11 0'0 2017-03-15 08:28:01.524045
0'0 2017-03-14 01:44:10.916908
0.29 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.086293 0'0 97:28 [8,4,9]
8 [8,4,9] 8 0'0 2017-03-14 05:40:30.224198
0'0 2017-03-12 22:50:44.393454
0.28 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:00:05.045972 0'0 99:52 [2,5,6]
2 [2,5,6] 2 0'0 2017-03-15 11:00:05.045921
0'0 2017-03-14 04:13:27.042516
0.27 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.085837 0'0 97:52 [4,5,10]
4 [4,5,10] 4 0'0 2017-03-15 07:42:35.345807
0'0 2017-03-14 06:58:32.545361
0.26 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:52:48.382356 0'0 100:43 [1,7,6]
1 [1,7,6] 1 0'0 2017-03-15 11:52:48.382303
0'0 2017-03-14 09:43:14.727045
0.25 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.121464 0'0 99:66 [3,1,11]
3 [3,1,11] 3 0'0 2017-03-15 10:38:54.423533
0'0 2017-03-13 23:40:26.077607
0.24 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:44.783790 0'0 93:24 [11,0,3]
11 [11,0,3] 11 0'0 2017-03-14 09:27:51.202384
0'0 2017-03-14 09:27:51.202384
0.23 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:45.799383 0'0 94:51 [3,8,0]
3 [3,8,0] 3 0'0 2017-03-15 04:59:02.254164
0'0 2017-03-12 22:50:44.393446
0.22 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.195410 0'0 99:39 [8,9,2]
8 [8,9,2] 8 0'0 2017-03-15 09:44:03.948601
0'0 2017-03-12 22:50:44.393445
0.21 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:00:36.528918 0'0 99:72 [3,10,1]
3 [3,10,1] 3 0'0 2017-03-15 11:00:36.528870
0'0 2017-03-12 22:50:44.393444
0.20 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:45.799609 0'0 94:38 [7,0,8]
7 [7,0,8] 7 0'0 2017-03-15 10:13:21.551419
0'0 2017-03-12 22:50:44.393442
0.1f 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:27:27.285609 0'0 100:54 [6,1,7]
6 [6,1,7] 6 0'0 2017-03-15 11:27:27.285530
0'0 2017-03-12 22:50:44.393441
0.1e 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:00:37.897933 0'0 99:39 [7,1,6]
7 [7,1,6] 7 0'0 2017-03-15 11:00:37.897892
0'0 2017-03-12 22:50:44.393440
0.1d 0 0 0 0 0 0 0
0 active+clean 2017-03-15 12:14:43.246527 0'0 100:29 [8,0,5]
8 [8,0,5] 8 0'0 2017-03-15 12:14:43.246470
0'0 2017-03-15 12:14:43.246470
0.1c 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:45.799019 0'0 94:63 [3,10,0]
3 [3,10,0] 3 0'0 2017-03-15 01:25:50.956186
0'0 2017-03-12 22:50:44.393437
0.1b 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.121571 0'0 99:39 [7,6,1]
7 [7,6,1] 7 0'0 2017-03-15 08:18:54.805154
0'0 2017-03-12 22:50:44.393436
0.1a 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.119244 0'0 99:48 [5,1,8]
5 [5,1,8] 5 0'0 2017-03-15 08:00:51.862520
0'0 2017-03-15 08:00:51.862520
0.19 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:44.784074 0'0 93:62 [0,11,7]
0 [0,11,7] 0 0'0 2017-03-15 07:57:13.935810
0'0 2017-03-12 22:50:44.393434
0.18 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:38:15.847932 0'0 100:30 [10,4,7]
10 [10,4,7] 10 0'0 2017-03-15 11:38:15.847796
0'0 2017-03-15 11:38:15.847796
0.17 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:44.784295 0'0 93:42 [0,5,8]
0 [0,5,8] 0 0'0 2017-03-14 05:05:08.239405
0'0 2017-03-12 22:50:44.393431
0.16 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.118421 0'0 99:30 [8,1,3]
8 [8,1,3] 8 0'0 2017-03-15 01:58:21.000099
0'0 2017-03-15 01:58:21.000099
3.a 3289 0 0 0 0 0 3028
3028 active+clean 2017-03-15 10:59:52.122364 100'67947 100:20697 [6,1,3]
6 [6,1,3] 6 85'60019 2017-03-15 06:37:03.261930
85'60019 2017-03-15 06:37:03.261930
0.9 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:59:38.409276 0'0 100:52 [1,7,8]
1 [1,7,8] 1 0'0 2017-03-15 11:59:38.409200
0'0 2017-03-12 22:50:44.393401
1.8 26893 0 0 0 0 4917872433 3062
3062 active+clean 2017-03-15 10:59:54.109028 100'264150 100:206321 [6,4,3]
6 [6,4,3] 6 85'201591 2017-03-14 09:54:17.628516
0'0 2017-03-12 22:58:51.551958
2.b 39708 0 0 0 0 26703758232 3043
3043 active+clean 2017-03-15 10:59:52.195669 100'53392 100:32320 [8,2,5]
8 [8,2,5] 8 85'44426 2017-03-15 10:02:33.753337
0'0 2017-03-12 22:58:53.643259
3.b 3227 0 0 0 0 0 3050
3050 active+clean 2017-03-15 11:28:27.671176 100'70617 100:24248 [10,4,7]
10 [10,4,7] 10 100'70610 2017-03-15 11:28:27.671107
85'57865 2017-03-14 08:57:23.371141
0.8 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.196853 0'0 99:58 [3,10,2]
3 [3,10,2] 3 0'0 2017-03-15 02:59:29.527039
0'0 2017-03-14 02:45:37.740896
1.9 26478 0 0 0 0 4906646660 3009
3009 active+clean 2017-03-15 10:59:57.382487 100'266677 100:222230 [8,3,2]
8 [8,3,2] 8 85'234486 2017-03-15 03:37:28.182922
0'0 2017-03-12 22:58:51.551964
2.a 39321 0 0 0 0 26438889950 3055
3055 active+clean 2017-03-15 10:59:52.194575 100'54185 100:34972 [11,7,2]
11 [11,7,2] 11 85'40485 2017-03-14 09:26:04.085556
0'0 2017-03-12 22:58:53.643255
5.2 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:45.788431 0'0 94:22 [0,3,8]
0 [0,3,8] 0 0'0 2017-03-15 08:50:51.367854
0'0 2017-03-12 22:59:15.463362
4.3 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:44.755072 0'0 93:19 [0,11,7]
0 [0,11,7] 0 0'0 2017-03-14 06:58:30.050414
0'0 2017-03-12 22:59:14.407022
0.7 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.113827 0'0 99:52 [1,11,7]
1 [1,11,7] 1 0'0 2017-03-15 02:36:48.848706
0'0 2017-03-12 22:50:44.393399
1.6 61802 0 0 0 0 9816799529 3024
3024 active+clean 2017-03-15 10:59:51.658259 100'576546 100:411873 [6,9,4]
6 [6,9,4] 6 85'500558 2017-03-15 08:11:10.883131
85'400651 2017-03-14 03:22:13.677665
2.5 82560 0 0 0 0 53753456206 3056
3056 active+clean 2017-03-15 11:28:09.997386 100'106323 100:57552 [8,1,5]
8 [8,1,5] 8 100'106261 2017-03-15 11:28:09.997315
85'78184 2017-03-14 06:22:40.250909
3.4 6621 0 0 0 0 8389230 3035
3035 active+clean 2017-03-15 10:59:58.242009 100'146755 100:61467 [7,8,2]
7 [7,8,2] 7 86'135686 2017-03-15 10:46:03.168639
0'0 2017-03-12 22:59:12.616738
5.3 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.120289 0'0 99:31 [9,1,11]
9 [9,1,11] 9 0'0 2017-03-15 06:57:38.870430
0'0 2017-03-14 01:55:43.658927
4.2 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.606935 0'0 99:18 [1,8,9]
1 [1,8,9] 1 0'0 2017-03-14 08:07:55.935833
0'0 2017-03-14 08:07:55.935833
0.6 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.086175 0'0 97:59 [4,11,5]
4 [4,11,5] 4 0'0 2017-03-14 08:07:53.069761
0'0 2017-03-12 22:50:44.393397
1.7 61031 0 0 0 0 10116071611 3004
3004 active+clean 2017-03-15 11:25:12.479773 100'550772 100:396565 [1,9,6]
1 [1,9,6] 1 100'547295 2017-03-15 11:25:12.479725
100'547295 2017-03-15 11:25:12.479725
2.4 80387 0 0 0 0 53220149134 3041
3041 active+clean 2017-03-15 11:02:23.117512 100'107407 100:64688 [5,10,4]
5 [5,10,4] 5 99'101538 2017-03-15 11:02:23.117442
0'0 2017-03-12 22:58:53.643225
3.5 6551 0 0 0 0 4194350 3017
3017 active+clean 2017-03-15 10:59:50.088771 100'138904 100:44971 [10,7,4]
10 [10,7,4] 10 85'125387 2017-03-15 03:41:18.534394
85'125387 2017-03-15 03:41:18.534394
5.0 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:00:10.046656 0'0 99:22 [2,10,5]
2 [2,10,5] 2 0'0 2017-03-15 11:00:10.046567
0'0 2017-03-15 11:00:10.046567
4.1 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:28:24.292278 0'0 100:28 [6,9,1]
6 [6,9,1] 6 0'0 2017-03-15 11:28:24.292221
0'0 2017-03-12 22:59:14.407011
0.5 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.086680 0'0 97:30 [10,4,5]
10 [10,4,5] 10 0'0 2017-03-15 08:34:52.484299
0'0 2017-03-12 22:50:44.393396
1.4 56928 0 0 0 0 9839309040 3049
3049 active+clean 2017-03-15 11:00:04.470324 100'531624 100:362602 [1,11,7]
1 [1,11,7] 1 85'414369 2017-03-14 10:49:35.949776
0'0 2017-03-12 22:58:51.551937
2.7 82322 0 0 0 0 53834805828 3047
3047 active+clean 2017-03-15 10:59:52.195096 100'104536 100:63955 [6,2,3]
6 [6,2,3] 6 85'78411 2017-03-14 09:38:09.990513
0'0 2017-03-12 22:58:53.643240
3.6 6532 0 0 0 0 2232513 3082
3082 active+clean 2017-03-15 10:59:52.120341 100'138906 100:54610 [5,6,1]
5 [5,6,1] 5 85'100301 2017-03-14 10:25:36.346634
0'0 2017-03-12 22:59:12.616750
5.1 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.087655 0'0 97:31 [11,7,4]
11 [11,7,4] 11 0'0 2017-03-15 09:41:14.016921
0'0 2017-03-12 22:59:15.463340
4.0 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:27:31.740731 0'0 100:27 [10,7,0]
10 [10,7,0] 10 0'0 2017-03-15 11:27:31.740654
0'0 2017-03-12 22:59:14.407004
0.4 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:01:58.543877 0'0 99:54 [3,6,4]
3 [3,6,4] 3 0'0 2017-03-15 11:01:58.543795
0'0 2017-03-12 22:50:44.393395
1.5 58177 0 0 0 0 10155300518 3012
3012 active+clean 2017-03-15 11:00:47.901456 100'524940 100:401847 [5,8,0]
5 [5,8,0] 5 99'502306 2017-03-15 11:00:47.901401
0'0 2017-03-12 22:58:51.551943
2.6 82467 0 0 0 0 52326091763 3015
3015 active+clean 2017-03-15 11:29:53.392379 100'102832 100:53079 [8,3,1]
8 [8,3,1] 8 100'102781 2017-03-15 11:29:53.392312
0'0 2017-03-12 22:58:53.643235
3.7 6550 0 0 0 0 16777770 3051
3051 active+clean 2017-03-15 11:27:32.528729 100'119971 100:45042 [8,9,4]
8 [8,9,4] 8 100'117836 2017-03-15 11:27:32.528657
0'0 2017-03-12 22:59:12.616753
5.6 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.196965 0'0 99:29 [7,11,2]
7 [7,11,2] 7 0'0 2017-03-14 07:18:11.475577
0'0 2017-03-12 22:59:15.463390
4.7 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.539380 0'0 99:18 [1,7,8]
1 [1,7,8] 1 0'0 2017-03-14 09:59:57.856166
0'0 2017-03-12 22:59:14.407040
0.3 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:35:42.366704 0'0 100:48 [5,4,10]
5 [5,4,10] 5 0'0 2017-03-15 11:35:42.366625
0'0 2017-03-12 22:50:44.393393
1.2 29771 0 0 0 0 4990621053 3036
3036 active+clean 2017-03-15 10:59:58.240268 100'263520 100:256811 [7,6,1]
7 [7,6,1] 7 85'228631 2017-03-15 02:16:52.733869
0'0 2017-03-12 22:58:51.551925
2.1 41092 0 0 0 0 27097439153 3038
3038 active+clean 2017-03-15 10:59:52.265330 100'53447 100:30297 [2,3,8]
2 [2,3,8] 2 85'44229 2017-03-15 07:26:32.606767
85'37031 2017-03-14 00:54:50.771691
3.0 3094 0 0 0 0 0 3095
3095 active+clean 2017-03-15 10:59:50.094902 100'59096 100:20746 [5,6,4]
5 [5,6,4] 5 85'43054 2017-03-14 05:12:42.118592
0'0 2017-03-12 22:59:12.616697
5.7 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:53.151396 0'0 99:22 [4,8,7]
4 [4,8,7] 4 0'0 2017-03-15 10:59:53.151304
0'0 2017-03-14 03:38:12.984983
4.6 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:45.796843 0'0 94:20 [0,3,10]
0 [0,3,10] 0 0'0 2017-03-15 08:50:44.367407
0'0 2017-03-12 22:59:14.407036
0.2 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.087220 0'0 97:39 [3,4,6]
3 [3,4,6] 3 0'0 2017-03-15 06:00:51.617930
0'0 2017-03-12 22:50:44.393392
1.3 30426 0 0 0 0 4975465340 3095
3095 active+clean 2017-03-15 11:00:03.517811 100'266637 100:166850 [3,10,1]
3 [3,10,1] 3 85'231830 2017-03-15 09:58:03.740174
0'0 2017-03-12 22:58:51.551930
2.0 40803 0 0 0 0 26221404597 3066
3066 active+clean 2017-03-15 10:59:52.193019 100'52830 100:31031 [2,10,7]
2 [2,10,7] 2 85'40136 2017-03-14 09:40:36.647549
0'0 2017-03-12 22:58:53.643202
3.1 3275 0 0 0 0 0 3088
3088 active+clean 2017-03-15 11:00:04.472750 100'73396 100:27940 [3,2,11]
3 [3,2,11] 3 85'61551 2017-03-15 03:50:33.520479
85'50888 2017-03-14 00:55:47.692695
5.4 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.123323 0'0 99:31 [10,5,1]
10 [10,5,1] 10 0'0 2017-03-15 09:21:52.870470
0'0 2017-03-14 02:44:15.992242
4.5 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.086549 0'0 97:26 [6,4,7]
6 [6,4,7] 6 0'0 2017-03-14 08:45:26.927014
0'0 2017-03-12 22:59:14.407030
0.1 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:00:34.527441 0'0 99:39 [3,6,1]
3 [3,6,1] 3 0'0 2017-03-15 11:00:34.527350
0'0 2017-03-12 22:50:44.393390
1.0 29838 0 0 0 0 5068704725 3014
3014 active+clean 2017-03-15 11:00:00.926181 100'282196 100:190123 [7,2,10]
7 [7,2,10] 7 85'247679 2017-03-15 03:38:02.649817
85'193471 2017-03-14 00:16:35.785265
2.3 40414 0 0 0 0 26633712199 3082
3082 active+clean 2017-03-15 11:27:36.722862 100'51993 100:27760 [11,5,1]
11 [11,5,1] 11 100'51963 2017-03-15 11:27:36.722787
0'0 2017-03-12 22:58:53.643218
3.2 3161 0 0 0 0 4194304 3070
3070 active+clean 2017-03-15 10:59:50.087049 100'68811 100:42765 [4,6,5]
4 [4,6,5] 4 85'52803 2017-03-15 09:37:00.943205
0'0 2017-03-12 22:59:12.616728
5.5 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:45.799044 0'0 94:28 [7,0,10]
7 [7,0,10] 7 0'0 2017-03-14 07:44:07.690644
0'0 2017-03-12 22:59:15.463384
4.4 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:44.784938 0'0 93:24 [5,10,0]
5 [5,10,0] 5 0'0 2017-03-15 08:44:42.159768
0'0 2017-03-14 04:58:49.481122
0.0 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.086403 0'0 97:70 [4,11,5]
4 [4,11,5] 4 0'0 2017-03-15 03:51:30.434269
0'0 2017-03-12 22:50:44.393379
1.1 29353 0 0 0 0 5040607354 3087
3087 active+clean 2017-03-15 11:00:00.474951 100'282930 100:159187 [10,7,1]
10 [10,7,1] 10 85'248719 2017-03-15 10:02:03.173127
85'248719 2017-03-15 10:02:03.173127
2.2 40836 0 0 0 0 27221671010 3027
3027 active+clean 2017-03-15 10:59:44.783666 100'52586 100:30004 [0,5,11]
0 [0,5,11] 0 85'40391 2017-03-14 11:08:04.337946
85'40391 2017-03-14 11:08:04.337946
3.3 3311 0 0 0 0 34 3041
3041 active+clean 2017-03-15 11:28:23.607721 100'65924 100:27358 [10,4,5]
10 [10,4,5] 10 100'65917 2017-03-15 11:28:23.607663
0'0 2017-03-12 22:59:12.616732
3.9 3309 0 0 0 0 8388608 3043
3043 active+clean 2017-03-15 10:59:50.085242 100'59533 100:34102 [4,5,10]
4 [4,5,10] 4 85'48687 2017-03-14 09:43:13.590688
0'0 2017-03-12 22:59:12.616762
0.a 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.194122 0'0 99:54 [6,7,2]
6 [6,7,2] 6 0'0 2017-03-15 09:32:30.428371
0'0 2017-03-15 09:32:30.428371
1.b 28192 0 0 0 0 4985785537 3052
3052 active+clean 2017-03-15 11:32:11.000959 100'262882 100:195337 [1,3,10]
1 [1,3,10] 1 100'262310 2017-03-15 11:32:11.000879
0'0 2017-03-12 22:58:51.551973
2.8 40224 0 0 0 0 26569935756 3082
3082 active+clean 2017-03-15 10:59:45.806257 100'54254 100:32405 [0,10,7]
0 [0,10,7] 0 85'39981 2017-03-14 04:59:12.951119
0'0 2017-03-12 22:58:53.643246
3.8 3355 0 0 0 0 4194304 3038
3038 active+clean 2017-03-15 10:59:50.087874 100'68609 100:28224 [11,7,4]
11 [11,7,4] 11 85'57531 2017-03-15 02:05:09.595286
0'0 2017-03-12 22:59:12.616757
0.b 0 0 0 0 0 0 0
0 active+clean 2017-03-15 12:43:34.813725 0'0 100:61 [4,9,8]
4 [4,9,8] 4 0'0 2017-03-15 12:43:34.813647
0'0 2017-03-12 22:50:44.393404
1.a 27562 0 0 0 0 5022517954 3096
3096 active+clean 2017-03-15 11:00:14.456373 100'262024 100:164259 [1,7,6]
1 [1,7,6] 1 99'251716 2017-03-15 11:00:14.456299
0'0 2017-03-12 22:58:51.551968
2.9 40024 0 0 0 0 26283858136 3071
3071 active+clean 2017-03-15 11:10:54.706535 100'54152 100:30993 [5,10,2]
5 [5,10,2] 5 100'51820 2017-03-15 11:10:54.706476
100'51820 2017-03-15 11:10:54.706476
5.9 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.540224 0'0 99:22 [1,3,11]
1 [1,3,11] 1 0'0 2017-03-15 02:59:29.046947
0'0 2017-03-12 22:59:15.463403
4.8 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.119633 0'0 99:20 [1,6,7]
1 [1,6,7] 1 0'0 2017-03-15 05:10:13.051843
0'0 2017-03-12 22:59:14.407045
0.c 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.120332 0'0 99:28 [5,6,1]
5 [5,6,1] 5 0'0 2017-03-14 04:52:28.427628
0'0 2017-03-12 22:50:44.393405
5.8 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:27:47.303985 0'0 100:31 [6,1,3]
6 [6,1,3] 6 0'0 2017-03-15 11:27:47.303913
0'0 2017-03-15 11:27:47.303913
4.9 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.540382 0'0 99:18 [1,7,6]
1 [1,7,6] 1 0'0 2017-03-14 10:23:02.048252
0'0 2017-03-12 22:59:14.407050
0.d 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.088091 0'0 97:39 [10,9,4]
10 [10,9,4] 10 0'0 2017-03-15 06:12:31.356825
0'0 2017-03-12 22:50:44.393406
5.b 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.193670 0'0 99:22 [2,3,8]
2 [2,3,8] 2 0'0 2017-03-15 08:17:40.715399
0'0 2017-03-12 22:59:15.463411
4.a 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:44.763882 0'0 93:24 [11,7,0]
11 [11,7,0] 11 0'0 2017-03-15 10:02:30.159822
0'0 2017-03-12 22:59:14.407055
0.e 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:54.146801 0'0 99:43 [4,5,8]
4 [4,5,8] 4 0'0 2017-03-15 10:59:54.146724
0'0 2017-03-14 04:21:15.290464
5.a 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.088314 0'0 97:29 [3,6,4]
3 [3,6,4] 3 0'0 2017-03-14 10:38:02.413221
0'0 2017-03-12 22:59:15.463407
4.b 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:49.486180 0'0 96:21 [0,11,9]
0 [0,11,9] 0 0'0 2017-03-15 10:59:49.486069
0'0 2017-03-13 23:44:23.802829
0.f 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.193296 0'0 99:41 [2,10,7]
2 [2,10,7] 2 0'0 2017-03-14 05:05:12.405058
0'0 2017-03-12 22:50:44.393409
0.10 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.085901 0'0 97:52 [4,8,3]
4 [4,8,3] 4 0'0 2017-03-15 04:55:45.934842
0'0 2017-03-12 22:50:44.393410
0.11 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:27:03.732936 0'0 100:61 [3,8,1]
3 [3,8,1] 3 0'0 2017-03-15 11:27:03.732878
0'0 2017-03-12 22:50:44.393411
0.12 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:00:33.956360 0'0 99:48 [9,6,1]
9 [9,6,1] 9 0'0 2017-03-15 11:00:33.956281
0'0 2017-03-14 00:37:12.089111
0.13 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.086980 0'0 97:39 [3,10,4]
3 [3,10,4] 3 0'0 2017-03-15 07:02:29.992516
0'0 2017-03-15 07:02:29.992516
0.14 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:55.011464 0'0 99:52 [1,11,5]
1 [1,11,5] 1 0'0 2017-03-15 10:59:55.011430
0'0 2017-03-12 22:50:44.393415
0.15 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.072892 0'0 97:50 [4,6,5]
4 [4,6,5] 4 0'0 2017-03-14 09:43:14.862695
0'0 2017-03-12 22:50:44.393417
0.2e 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:57.044595 0'0 99:52 [2,7,8]
2 [2,7,8] 2 0'0 2017-03-15 10:59:57.044557
0'0 2017-03-12 22:50:44.393472
0.2f 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.195618 0'0 99:28 [11,7,2]
11 [11,7,2] 11 0'0 2017-03-14 09:04:49.071489
0'0 2017-03-12 22:50:44.393473
0.30 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.122884 0'0 99:42 [8,1,9]
8 [8,1,9] 8 0'0 2017-03-15 08:23:54.467250
0'0 2017-03-12 22:50:44.393474
0.31 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.540619 0'0 99:50 [1,11,3]
1 [1,11,3] 1 0'0 2017-03-14 09:40:26.722867
0'0 2017-03-12 22:50:44.393476
0.32 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.194257 0'0 99:61 [2,11,7]
2 [2,11,7] 2 0'0 2017-03-15 07:35:23.370607
0'0 2017-03-12 22:50:44.393477
0.33 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.196550 0'0 99:37 [3,10,2]
3 [3,10,2] 3 0'0 2017-03-14 10:22:36.232867
0'0 2017-03-12 22:50:44.393478
0.34 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:00:36.960876 0'0 99:48 [9,11,2]
9 [9,11,2] 9 0'0 2017-03-15 11:00:36.960827
0'0 2017-03-12 22:50:44.393479
0.35 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.119940 0'0 99:46 [5,6,1]
5 [5,6,1] 5 0'0 2017-03-14 05:38:36.747488
0'0 2017-03-12 22:50:44.393481
0.36 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.087236 0'0 97:48 [9,6,4]
9 [9,6,4] 9 0'0 2017-03-15 09:17:58.121500
0'0 2017-03-12 22:50:44.393482
0.37 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.194785 0'0 99:39 [9,2,8]
9 [9,2,8] 9 0'0 2017-03-15 04:47:09.503797
0'0 2017-03-12 22:50:44.393483
0.38 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:45.798599 0'0 94:56 [9,0,11]
9 [9,0,11] 9 0'0 2017-03-15 04:44:51.485759
0'0 2017-03-12 22:50:44.393484
0.39 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:52:52.392420 0'0 100:48 [9,1,6]
9 [9,1,6] 9 0'0 2017-03-15 11:52:52.392352
0'0 2017-03-12 22:50:44.393486
0.3a 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:50.086627 0'0 97:59 [4,8,7]
4 [4,8,7] 4 0'0 2017-03-14 09:43:11.863773
0'0 2017-03-14 09:43:11.863773
0.3b 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.171056 0'0 99:43 [2,8,5]
2 [2,8,5] 2 0'0 2017-03-15 07:41:47.424549
0'0 2017-03-12 22:50:44.393488
0.3c 0 0 0 0 0 0 0
0 active+clean 2017-03-15 11:02:00.981332 0'0 99:39 [9,2,8]
9 [9,2,8] 9 0'0 2017-03-15 11:02:00.981264
0'0 2017-03-12 22:50:44.393490
0.3d 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.540831 0'0 99:61 [1,10,7]
1 [1,10,7] 1 0'0 2017-03-15 07:55:33.313612
0'0 2017-03-12 22:50:44.393491
0.3e 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.195787 0'0 99:30 [8,2,3]
8 [8,2,3] 8 0'0 2017-03-15 02:28:17.185544
0'0 2017-03-12 22:50:44.393492
0.3f 0 0 0 0 0 0 0
0 active+clean 2017-03-15 10:59:52.265530 0'0 99:43 [2,11,5]
2 [2,11,5] 2 0'0 2017-03-15 00:30:38.822799
0'0 2017-03-12 22:50:44.393493
5 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0
3 52275 0 0 0 0 48371113 36638 36638
2 650158 0 0 0 0 426305171964 36623 36623
1 466451 0 0 0 0 79835701754 36540 36540
0 0 0 0 0 0 0 0 0
sum 1168884 0 0 0 0 506189244831 109801 109801
OSD_STAT USED AVAIL TOTAL HB_PEERS PG_SUM
11 56284M 223G 278G [0,1,2,3,4,5,7,8,9,10] 28
10 97062M 183G 278G [0,1,2,4,5,6,7,8,9,11] 30
0 41564M 238G 278G [1,3,4,5,6,7,8,9,10,11] 22
1 123G 154G 278G [0,2,3,5,6,7,8,9,10,11] 42
2 112G 166G 278G [1,3,4,5,6,7,8,9,10,11] 28
4 47643M 232G 278G [0,1,3,5,6,7,8,9,10,11] 32
3 97557M 183G 278G [0,1,2,4,6,7,8,9,10,11] 32
5 127G 151G 278G [0,1,2,4,6,7,8,9,10,11] 31
6 71151M 209G 278G [1,2,3,4,5,7,8,9,10,11] 32
7 79459M 201G 278G [0,1,2,4,5,6,8,9,10,11] 40
9 23961M 112G 136G [0,1,2,4,5,6,7,8,10,11] 21
8 104G 174G 278G [0,1,2,3,4,5,7,9,10,11] 34
sum 970G 2231G 3202G
Rhian Resnick
Assistant Director Middleware and HPC
Office of Information Technology
Florida Atlantic University
777 Glades Road, CM22, Rm 173B
Boca Raton, FL 33431
Phone 561.297.2647
Fax 561.297.0222
[image] <https://hpc.fau.edu/wp-content/uploads/2015/01/image.jpg>
_______________________________________________
ceph-users mailing list
[email protected]
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
