Hi, There is a cluster of ceph. To accelerate dokinuli ssd into it, the card: I ssd gathered in a separate pool and a prescribed rule. But I'm getting the following error msg:
[ceph@ceph-adm test]$ ceph -s cluster a73f8be5-0fa6-4a82-aa9b-22121246398e health HEALTH_WARN 64 pgs stuck inactive 64 pgs stuck unclean monmap e1: 3 mons at {srv-osd01=10.30.15.20:6789/0, srv-osd02=10.30.15.21:6789/0,srv-osd03=10.30.15.22:6789/0} election epoch 76, quorum 0,1,2 srv-osd01,srv-osd02,srv-osd03 osdmap e9500: 135 osds: 135 up, 135 in flags sortbitwise pgmap v16447007: 4168 pgs, 3 pools, 5829 GB data, 1488 kobjects 17469 GB used, 270 TB / 287 TB avail 4104 active+clean 64 creating client io 179 kB/s rd, 8781 kB/s wr, 177 op/s If you change the value of ruleset 0 to ruleset 2 all will work. Possibly due to the removal of osd.52? ceph.conf [global] fsid = a73f8be5-0fa6-4a82-aa9b-22121246398e mon_initial_members = srv-osd01, srv-osd02, srv-osd03 mon_host = 10.30.15.20,10.30.15.21,10.30.15.22 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx filestore_xattr_use_omap = true public network = 10.30.15.0/24 cluster network = 10.40.10.0/24 osd_crush_update_on_start = false My crushrule map: [ceph@ceph-adm test]$ cat map.decompile # begin crush map tunable choose_local_tries 0 tunable choose_local_fallback_tries 0 tunable choose_total_tries 50 tunable chooseleaf_descend_once 1 tunable straw_calc_version 1 # devices device 0 osd.0 device 1 osd.1 device 2 osd.2 device 3 osd.3 device 4 osd.4 device 5 osd.5 device 6 osd.6 device 7 osd.7 device 8 osd.8 device 9 osd.9 device 10 osd.10 device 11 osd.11 device 12 osd.12 device 13 osd.13 device 14 osd.14 device 15 osd.15 device 16 osd.16 device 17 osd.17 device 18 osd.18 device 19 osd.19 device 20 osd.20 device 21 osd.21 device 22 osd.22 device 23 osd.23 device 24 osd.24 device 25 osd.25 device 26 osd.26 device 27 osd.27 device 28 osd.28 device 29 osd.29 device 30 osd.30 device 31 osd.31 device 32 osd.32 device 33 osd.33 device 34 osd.34 device 35 osd.35 device 36 osd.36 device 37 osd.37 device 38 osd.38 device 39 osd.39 device 40 osd.40 device 41 osd.41 device 42 osd.42 device 43 osd.43 device 44 osd.44 device 45 osd.45 device 46 osd.46 device 47 osd.47 device 48 osd.48 device 49 osd.49 device 50 osd.50 device 51 osd.51 device 52 device52 device 53 osd.53 device 54 osd.54 device 55 osd.55 device 56 osd.56 device 57 osd.57 device 58 osd.58 device 59 osd.59 device 60 osd.60 device 61 osd.61 device 62 osd.62 device 63 osd.63 device 64 osd.64 device 65 osd.65 device 66 osd.66 device 67 osd.67 device 68 osd.68 device 69 osd.69 device 70 osd.70 device 71 osd.71 device 72 osd.72 device 73 osd.73 device 74 osd.74 device 75 osd.75 device 76 osd.76 device 77 osd.77 device 78 osd.78 device 79 osd.79 device 80 osd.80 device 81 osd.81 device 82 osd.82 device 83 osd.83 device 84 osd.84 device 85 osd.85 device 86 osd.86 device 87 osd.87 device 88 osd.88 device 89 osd.89 device 90 osd.90 device 91 osd.91 device 92 osd.92 device 93 osd.93 device 94 osd.94 device 95 osd.95 device 96 osd.96 device 97 osd.97 device 98 osd.98 device 99 osd.99 device 100 osd.100 device 101 osd.101 device 102 osd.102 device 103 osd.103 device 104 osd.104 device 105 osd.105 device 106 osd.106 device 107 osd.107 device 108 osd.108 device 109 osd.109 device 110 osd.110 device 111 osd.111 device 112 osd.112 device 113 osd.113 device 114 osd.114 device 115 osd.115 device 116 osd.116 device 117 osd.117 device 118 osd.118 device 119 osd.119 device 120 osd.120 device 121 osd.121 device 122 osd.122 device 123 osd.123 device 124 osd.124 device 125 osd.125 device 126 osd.126 device 127 osd.127 device 128 osd.128 device 129 osd.129 device 130 osd.130 device 131 osd.131 device 132 osd.132 device 133 osd.133 device 134 osd.134 device 135 osd.135 # types type 0 osd type 1 host type 2 chassis type 3 rack type 4 row type 5 pdu type 6 pod type 7 room type 8 datacenter type 9 region type 10 root # buckets host srv-osd01-ssd { id -2 # do not change unnecessarily # weight 0.854 alg straw hash 0 # rjenkins1 item osd.0 weight 0.427 item osd.1 weight 0.427 } host srv-osd02-ssd { id -3 # do not change unnecessarily # weight 0.854 alg straw hash 0 # rjenkins1 item osd.2 weight 0.427 item osd.3 weight 0.427 } host srv-osd03-ssd { id -4 # do not change unnecessarily # weight 0.854 alg straw hash 0 # rjenkins1 item osd.4 weight 0.427 item osd.5 weight 0.427 } host srv-osd04-ssd { id -5 # do not change unnecessarily # weight 0.854 alg straw hash 0 # rjenkins1 item osd.6 weight 0.427 item osd.7 weight 0.427 } host srv-osd05-ssd { id -6 # do not change unnecessarily # weight 0.344 alg straw hash 0 # rjenkins1 item osd.135 weight 0.172 item osd.124 weight 0.172 } host srv-osd06-ssd { id -7 # do not change unnecessarily # weight 0.344 alg straw hash 0 # rjenkins1 item osd.48 weight 0.172 item osd.49 weight 0.172 } host srv-osd07-ssd { id -8 # do not change unnecessarily # weight 0.344 alg straw hash 0 # rjenkins1 item osd.50 weight 0.172 item osd.51 weight 0.172 } host srv-osd08-ssd { id -9 # do not change unnecessarily # weight 0.172 alg straw hash 0 # rjenkins1 item osd.53 weight 0.172 } root ssd { id -1 # do not change unnecessarily # weight 4.530 alg straw hash 0 # rjenkins1 item srv-osd01-ssd weight 0.854 item srv-osd02-ssd weight 0.854 item srv-osd03-ssd weight 0.854 item srv-osd04-ssd weight 0.854 item srv-osd05-ssd weight 0.314 item srv-osd06-ssd weight 0.314 item srv-osd07-ssd weight 0.314 item srv-osd08-ssd weight 0.172 } host srv-osd01-hdd { id -102 # do not change unnecessarily # weight 54.470 alg straw hash 0 # rjenkins1 item osd.8 weight 5.447 item osd.9 weight 5.447 item osd.10 weight 5.447 item osd.11 weight 5.447 item osd.12 weight 5.447 item osd.13 weight 5.447 item osd.14 weight 5.447 item osd.15 weight 5.447 item osd.16 weight 5.447 item osd.17 weight 5.447 } host srv-osd02-hdd { id -103 # do not change unnecessarily # weight 54.470 alg straw hash 0 # rjenkins1 item osd.18 weight 5.447 item osd.19 weight 5.447 item osd.20 weight 5.447 item osd.21 weight 5.447 item osd.22 weight 5.447 item osd.23 weight 5.447 item osd.24 weight 5.447 item osd.25 weight 5.447 item osd.26 weight 5.447 item osd.46 weight 5.447 } host srv-osd03-hdd { id -104 # do not change unnecessarily # weight 54.470 alg straw hash 0 # rjenkins1 item osd.27 weight 5.447 item osd.28 weight 5.447 item osd.29 weight 5.447 item osd.30 weight 5.447 item osd.31 weight 5.447 item osd.32 weight 5.447 item osd.33 weight 5.447 item osd.34 weight 5.447 item osd.35 weight 5.447 item osd.36 weight 5.447 } host srv-osd04-hdd { id -105 # do not change unnecessarily # weight 54.470 alg straw hash 0 # rjenkins1 item osd.37 weight 5.447 item osd.38 weight 5.447 item osd.39 weight 5.447 item osd.40 weight 5.447 item osd.41 weight 5.447 item osd.42 weight 5.447 item osd.43 weight 5.447 item osd.44 weight 5.447 item osd.45 weight 5.447 item osd.47 weight 5.447 } host srv-osd05-hdd { id -106 # do not change unnecessarily # weight 16.160 alg straw hash 0 # rjenkins1 item osd.114 weight 0.808 item osd.115 weight 0.808 item osd.116 weight 0.808 item osd.117 weight 0.808 item osd.118 weight 0.808 item osd.119 weight 0.808 item osd.120 weight 0.808 item osd.121 weight 0.808 item osd.122 weight 0.808 item osd.123 weight 0.808 item osd.125 weight 0.808 item osd.126 weight 0.808 item osd.127 weight 0.808 item osd.128 weight 0.808 item osd.129 weight 0.808 item osd.130 weight 0.808 item osd.131 weight 0.808 item osd.132 weight 0.808 item osd.133 weight 0.808 item osd.134 weight 0.808 } host srv-osd06-hdd { id -107 # do not change unnecessarily # weight 16.160 alg straw hash 0 # rjenkins1 item osd.55 weight 0.808 item osd.56 weight 0.808 item osd.57 weight 0.808 item osd.58 weight 0.808 item osd.59 weight 0.808 item osd.60 weight 0.808 item osd.61 weight 0.808 item osd.62 weight 0.808 item osd.63 weight 0.808 item osd.64 weight 0.808 item osd.65 weight 0.808 item osd.66 weight 0.808 item osd.67 weight 0.808 item osd.68 weight 0.808 item osd.69 weight 0.808 item osd.70 weight 0.808 item osd.71 weight 0.808 item osd.72 weight 0.808 item osd.73 weight 0.808 item osd.54 weight 0.808 } host srv-osd07-hdd { id -108 # do not change unnecessarily # weight 16.160 alg straw hash 0 # rjenkins1 item osd.74 weight 0.808 item osd.75 weight 0.808 item osd.76 weight 0.808 item osd.77 weight 0.808 item osd.78 weight 0.808 item osd.79 weight 0.808 item osd.80 weight 0.808 item osd.81 weight 0.808 item osd.82 weight 0.808 item osd.83 weight 0.808 item osd.84 weight 0.808 item osd.85 weight 0.808 item osd.86 weight 0.808 item osd.87 weight 0.808 item osd.88 weight 0.808 item osd.89 weight 0.808 item osd.90 weight 0.808 item osd.91 weight 0.808 item osd.92 weight 0.808 item osd.93 weight 0.808 } host srv-osd08-hdd { id -109 # do not change unnecessarily # weight 16.160 alg straw hash 0 # rjenkins1 item osd.94 weight 0.808 item osd.95 weight 0.808 item osd.96 weight 0.808 item osd.97 weight 0.808 item osd.98 weight 0.808 item osd.99 weight 0.808 item osd.100 weight 0.808 item osd.101 weight 0.808 item osd.102 weight 0.808 item osd.103 weight 0.808 item osd.104 weight 0.808 item osd.105 weight 0.808 item osd.106 weight 0.808 item osd.107 weight 0.808 item osd.108 weight 0.808 item osd.109 weight 0.808 item osd.110 weight 0.808 item osd.111 weight 0.808 item osd.112 weight 0.808 item osd.113 weight 0.808 } root hdd { id -100 # do not change unnecessarily # weight 282.632 alg straw hash 0 # rjenkins1 item srv-osd01-hdd weight 54.466 item srv-osd02-hdd weight 54.466 item srv-osd03-hdd weight 54.466 item srv-osd04-hdd weight 54.466 item srv-osd05-hdd weight 16.192 item srv-osd06-hdd weight 16.192 item srv-osd07-hdd weight 16.192 item srv-osd08-hdd weight 16.192 } # rules rule ssd { ruleset 0 type replicated min_size 1 max_size 10 step take ssd step chooseleaf firstn 0 type host step emit } rule hdd { ruleset 1 type replicated min_size 1 max_size 10 step take hdd step chooseleaf firstn 0 type host step emit } # end crush map [ceph@ceph-adm test]$ ceph osd tree ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY -100 282.63190 root hdd -102 54.46599 host srv-osd01-hdd 8 5.44699 osd.8 up 1.00000 1.00000 9 5.44699 osd.9 up 1.00000 1.00000 10 5.44699 osd.10 up 1.00000 1.00000 11 5.44699 osd.11 up 1.00000 1.00000 12 5.44699 osd.12 up 1.00000 1.00000 13 5.44699 osd.13 up 1.00000 1.00000 14 5.44699 osd.14 up 1.00000 1.00000 15 5.44699 osd.15 up 1.00000 1.00000 16 5.44699 osd.16 up 1.00000 1.00000 17 5.44699 osd.17 up 1.00000 1.00000 -103 54.46599 host srv-osd02-hdd 18 5.44699 osd.18 up 1.00000 1.00000 19 5.44699 osd.19 up 1.00000 1.00000 20 5.44699 osd.20 up 1.00000 1.00000 21 5.44699 osd.21 up 1.00000 1.00000 22 5.44699 osd.22 up 1.00000 1.00000 23 5.44699 osd.23 up 1.00000 1.00000 24 5.44699 osd.24 up 1.00000 1.00000 25 5.44699 osd.25 up 1.00000 1.00000 26 5.44699 osd.26 up 1.00000 1.00000 46 5.44699 osd.46 up 1.00000 1.00000 -104 54.46599 host srv-osd03-hdd 27 5.44699 osd.27 up 1.00000 1.00000 28 5.44699 osd.28 up 1.00000 1.00000 29 5.44699 osd.29 up 1.00000 1.00000 30 5.44699 osd.30 up 1.00000 1.00000 31 5.44699 osd.31 up 1.00000 1.00000 32 5.44699 osd.32 up 1.00000 1.00000 33 5.44699 osd.33 up 1.00000 1.00000 34 5.44699 osd.34 up 1.00000 1.00000 35 5.44699 osd.35 up 1.00000 1.00000 36 5.44699 osd.36 up 1.00000 1.00000 -105 54.46599 host srv-osd04-hdd 37 5.44699 osd.37 up 1.00000 1.00000 38 5.44699 osd.38 up 1.00000 1.00000 39 5.44699 osd.39 up 1.00000 1.00000 40 5.44699 osd.40 up 1.00000 1.00000 41 5.44699 osd.41 up 1.00000 1.00000 42 5.44699 osd.42 up 1.00000 1.00000 43 5.44699 osd.43 up 1.00000 1.00000 44 5.44699 osd.44 up 1.00000 1.00000 45 5.44699 osd.45 up 1.00000 1.00000 47 5.44699 osd.47 up 1.00000 1.00000 -106 16.19199 host srv-osd05-hdd 114 0.80800 osd.114 up 1.00000 1.00000 115 0.80800 osd.115 up 0.81648 1.00000 116 0.80800 osd.116 up 1.00000 1.00000 117 0.80800 osd.117 up 1.00000 1.00000 118 0.80800 osd.118 up 0.70801 1.00000 119 0.80800 osd.119 up 1.00000 1.00000 120 0.80800 osd.120 up 1.00000 1.00000 121 0.80800 osd.121 up 1.00000 1.00000 122 0.80800 osd.122 up 1.00000 1.00000 123 0.80800 osd.123 up 1.00000 1.00000 125 0.80800 osd.125 up 1.00000 1.00000 126 0.80800 osd.126 up 1.00000 1.00000 127 0.80800 osd.127 up 1.00000 1.00000 128 0.80800 osd.128 up 1.00000 1.00000 129 0.80800 osd.129 up 1.00000 1.00000 130 0.80800 osd.130 up 0.79892 1.00000 131 0.80800 osd.131 up 1.00000 1.00000 132 0.80800 osd.132 up 1.00000 1.00000 133 0.80800 osd.133 up 1.00000 1.00000 134 0.80800 osd.134 up 1.00000 1.00000 -107 16.19199 host srv-osd06-hdd 55 0.80800 osd.55 up 1.00000 1.00000 56 0.80800 osd.56 up 0.82719 1.00000 57 0.80800 osd.57 up 1.00000 1.00000 58 0.80800 osd.58 up 1.00000 1.00000 59 0.80800 osd.59 up 1.00000 1.00000 60 0.80800 osd.60 up 1.00000 1.00000 61 0.80800 osd.61 up 1.00000 1.00000 62 0.80800 osd.62 up 1.00000 1.00000 63 0.80800 osd.63 up 0.75986 1.00000 64 0.80800 osd.64 up 1.00000 1.00000 65 0.80800 osd.65 up 1.00000 1.00000 66 0.80800 osd.66 up 1.00000 1.00000 67 0.80800 osd.67 up 1.00000 1.00000 68 0.80800 osd.68 up 1.00000 1.00000 69 0.80800 osd.69 up 1.00000 1.00000 70 0.80800 osd.70 up 0.81743 1.00000 71 0.80800 osd.71 up 1.00000 1.00000 72 0.80800 osd.72 up 1.00000 1.00000 73 0.80800 osd.73 up 1.00000 1.00000 54 0.80800 osd.54 up 1.00000 1.00000 -108 16.19199 host srv-osd07-hdd 74 0.80800 osd.74 up 0.73679 1.00000 75 0.80800 osd.75 up 1.00000 1.00000 76 0.80800 osd.76 up 1.00000 1.00000 77 0.80800 osd.77 up 1.00000 1.00000 78 0.80800 osd.78 up 1.00000 1.00000 79 0.80800 osd.79 up 0.86931 1.00000 80 0.80800 osd.80 up 1.00000 1.00000 81 0.80800 osd.81 up 1.00000 1.00000 82 0.80800 osd.82 up 1.00000 1.00000 83 0.80800 osd.83 up 0.82845 1.00000 84 0.80800 osd.84 up 1.00000 1.00000 85 0.80800 osd.85 up 1.00000 1.00000 86 0.80800 osd.86 up 1.00000 1.00000 87 0.80800 osd.87 up 0.85548 1.00000 88 0.80800 osd.88 up 1.00000 1.00000 89 0.80800 osd.89 up 1.00000 1.00000 90 0.80800 osd.90 up 1.00000 1.00000 91 0.80800 osd.91 up 0.83315 1.00000 92 0.80800 osd.92 up 1.00000 1.00000 93 0.80800 osd.93 up 1.00000 1.00000 -109 16.19199 host srv-osd08-hdd 94 0.80800 osd.94 up 0.83136 1.00000 95 0.80800 osd.95 up 1.00000 1.00000 96 0.80800 osd.96 up 1.00000 1.00000 97 0.80800 osd.97 up 1.00000 1.00000 98 0.80800 osd.98 up 1.00000 1.00000 99 0.80800 osd.99 up 1.00000 1.00000 100 0.80800 osd.100 up 1.00000 1.00000 101 0.80800 osd.101 up 1.00000 1.00000 102 0.80800 osd.102 up 1.00000 1.00000 103 0.80800 osd.103 up 1.00000 1.00000 104 0.80800 osd.104 up 1.00000 1.00000 105 0.80800 osd.105 up 1.00000 1.00000 106 0.80800 osd.106 up 1.00000 1.00000 107 0.80800 osd.107 up 1.00000 1.00000 108 0.80800 osd.108 up 1.00000 1.00000 109 0.80800 osd.109 up 0.85403 1.00000 110 0.80800 osd.110 up 0.68864 1.00000 111 0.80800 osd.111 up 0.82031 1.00000 112 0.80800 osd.112 up 1.00000 1.00000 113 0.80800 osd.113 up 1.00000 1.00000 -1 4.52994 root ssd -2 0.85399 host srv-osd01-ssd 0 0.42699 osd.0 up 0.00015 1.00000 1 0.42699 osd.1 up 0.00012 1.00000 -3 0.85399 host srv-osd02-ssd 2 0.42699 osd.2 up 0.00014 1.00000 3 0.42699 osd.3 up 0.00018 1.00000 -4 0.85399 host srv-osd03-ssd 4 0.42699 osd.4 up 0.00012 1.00000 5 0.42699 osd.5 up 0.00015 1.00000 -5 0.85399 host srv-osd04-ssd 6 0.42699 osd.6 up 0.00018 1.00000 7 0.42699 osd.7 up 0.00026 1.00000 -6 0.31400 host srv-osd05-ssd 135 0.17200 osd.135 up 0.00006 1.00000 124 0.17200 osd.124 up 0.00021 1.00000 -7 0.31400 host srv-osd06-ssd 48 0.17200 osd.48 up 0.00006 1.00000 49 0.17200 osd.49 up 0.00014 1.00000 -8 0.31400 host srv-osd07-ssd 50 0.17200 osd.50 up 0.00014 1.00000 51 0.17200 osd.51 up 0.00018 1.00000 -9 0.17200 host srv-osd08-ssd 53 0.17200 osd.53 up 0.00005 1.00000
_______________________________________________ ceph-users mailing list ceph-users@lists.ceph.com http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com