Hi Paul,
thanks for your reply.
Wed, 19 Jun 2019 13:19:55 +0200
Paul Emmerich <[email protected]> ==> Lars Täuber <[email protected]> :
> Wild guess: you hit the PG hard limit, how many PGs per OSD do you have?
> If this is the case: increase "osd max pg per osd hard ratio"
>
> Check "ceph pg <pgid> query" to see why it isn't activating.
>
> Can you share the output of "ceph osd df tree" and "ceph pg <pgid> query"
> of the affected PGs?
The pg queries are attached. I can't read them - to much information.
Here is the osd df tree:
# osd df tree
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL
%USE VAR PGS STATUS TYPE NAME
-1 167.15057 - 167 TiB 4.7 TiB 1.2 TiB 952 MiB 57 GiB 162 TiB
2.79 1.00 - root PRZ
-17 72.43192 - 72 TiB 2.0 TiB 535 GiB 393 MiB 25 GiB 70 TiB
2.78 1.00 - rack 1-eins
-9 22.28674 - 22 TiB 640 GiB 170 GiB 82 MiB 9.0 GiB 22 TiB
2.80 1.01 - host onode1
2 hdd 5.57169 1.00000 5.6 TiB 162 GiB 45 GiB 11 MiB 2.3 GiB 5.4 TiB
2.84 1.02 224 up osd.2
9 hdd 5.57169 1.00000 5.6 TiB 156 GiB 39 GiB 19 MiB 2.1 GiB 5.4 TiB
2.74 0.98 201 up osd.9
14 hdd 5.57169 1.00000 5.6 TiB 162 GiB 44 GiB 24 MiB 2.1 GiB 5.4 TiB
2.84 1.02 230 up osd.14
21 hdd 5.57169 1.00000 5.6 TiB 160 GiB 42 GiB 27 MiB 2.5 GiB 5.4 TiB
2.80 1.00 219 up osd.21
-13 22.28674 - 22 TiB 640 GiB 170 GiB 123 MiB 8.9 GiB 22 TiB
2.80 1.00 - host onode4
4 hdd 5.57169 1.00000 5.6 TiB 156 GiB 39 GiB 38 MiB 2.2 GiB 5.4 TiB
2.73 0.98 205 up osd.4
11 hdd 5.57169 1.00000 5.6 TiB 164 GiB 47 GiB 24 MiB 2.0 GiB 5.4 TiB
2.87 1.03 241 up osd.11
18 hdd 5.57169 1.00000 5.6 TiB 159 GiB 42 GiB 31 MiB 2.5 GiB 5.4 TiB
2.79 1.00 221 up osd.18
22 hdd 5.57169 1.00000 5.6 TiB 160 GiB 43 GiB 29 MiB 2.1 GiB 5.4 TiB
2.81 1.01 225 up osd.22
-5 27.85843 - 28 TiB 782 GiB 195 GiB 188 MiB 6.9 GiB 27 TiB
2.74 0.98 - host onode7
5 hdd 5.57169 1.00000 5.6 TiB 158 GiB 41 GiB 26 MiB 1.2 GiB 5.4 TiB
2.77 0.99 213 up osd.5
12 hdd 5.57169 1.00000 5.6 TiB 159 GiB 42 GiB 31 MiB 993 MiB 5.4 TiB
2.79 1.00 222 up osd.12
20 hdd 5.57169 1.00000 5.6 TiB 157 GiB 40 GiB 47 MiB 1.2 GiB 5.4 TiB
2.76 0.99 212 up osd.20
27 hdd 5.57169 1.00000 5.6 TiB 151 GiB 33 GiB 28 MiB 1.9 GiB 5.4 TiB
2.64 0.95 179 up osd.27
29 hdd 5.57169 1.00000 5.6 TiB 156 GiB 39 GiB 56 MiB 1.7 GiB 5.4 TiB
2.74 0.98 203 up osd.29
-18 44.57349 - 45 TiB 1.3 TiB 341 GiB 248 MiB 14 GiB 43 TiB
2.81 1.01 - rack 2-zwei
-7 22.28674 - 22 TiB 641 GiB 171 GiB 132 MiB 6.7 GiB 22 TiB
2.81 1.01 - host onode2
1 hdd 5.57169 1.00000 5.6 TiB 155 GiB 38 GiB 35 MiB 1.2 GiB 5.4 TiB
2.72 0.97 203 up osd.1
8 hdd 5.57169 1.00000 5.6 TiB 163 GiB 46 GiB 36 MiB 2.4 GiB 5.4 TiB
2.86 1.02 243 up osd.8
16 hdd 5.57169 1.00000 5.6 TiB 161 GiB 43 GiB 24 MiB 1000 MiB 5.4 TiB
2.82 1.01 221 up osd.16
23 hdd 5.57169 1.00000 5.6 TiB 162 GiB 45 GiB 37 MiB 2.1 GiB 5.4 TiB
2.84 1.02 228 up osd.23
-3 22.28674 - 22 TiB 640 GiB 170 GiB 116 MiB 7.6 GiB 22 TiB
2.80 1.00 - host onode5
3 hdd 5.57169 1.00000 5.6 TiB 154 GiB 36 GiB 14 MiB 1010 MiB 5.4 TiB
2.70 0.97 186 up osd.3
7 hdd 5.57169 1.00000 5.6 TiB 161 GiB 44 GiB 22 MiB 2.2 GiB 5.4 TiB
2.82 1.01 221 up osd.7
15 hdd 5.57169 1.00000 5.6 TiB 165 GiB 48 GiB 26 MiB 2.3 GiB 5.4 TiB
2.89 1.04 249 up osd.15
24 hdd 5.57169 1.00000 5.6 TiB 160 GiB 42 GiB 54 MiB 2.1 GiB 5.4 TiB
2.80 1.00 223 up osd.24
-19 50.14517 - 50 TiB 1.4 TiB 376 GiB 311 MiB 18 GiB 49 TiB
2.79 1.00 - rack 3-drei
-15 22.28674 - 22 TiB 649 GiB 179 GiB 112 MiB 8.2 GiB 22 TiB
2.84 1.02 - host onode3
0 hdd 5.57169 1.00000 5.6 TiB 162 GiB 45 GiB 28 MiB 996 MiB 5.4 TiB
2.84 1.02 229 up osd.0
10 hdd 5.57169 1.00000 5.6 TiB 159 GiB 42 GiB 21 MiB 2.2 GiB 5.4 TiB
2.79 1.00 213 up osd.10
17 hdd 5.57169 1.00000 5.6 TiB 165 GiB 47 GiB 19 MiB 2.5 GiB 5.4 TiB
2.88 1.03 238 up osd.17
25 hdd 5.57169 1.00000 5.6 TiB 163 GiB 46 GiB 44 MiB 2.5 GiB 5.4 TiB
2.86 1.03 242 up osd.25
-11 27.85843 - 28 TiB 784 GiB 197 GiB 199 MiB 9.4 GiB 27 TiB
2.75 0.99 - host onode6
6 hdd 5.57169 1.00000 5.6 TiB 158 GiB 40 GiB 22 MiB 2.0 GiB 5.4 TiB
2.76 0.99 210 up osd.6
13 hdd 5.57169 1.00000 5.6 TiB 155 GiB 38 GiB 42 MiB 2.4 GiB 5.4 TiB
2.73 0.98 194 up osd.13
19 hdd 5.57169 1.00000 5.6 TiB 161 GiB 43 GiB 53 MiB 971 MiB 5.4 TiB
2.82 1.01 241 up osd.19
26 hdd 5.57169 1.00000 5.6 TiB 155 GiB 37 GiB 34 MiB 2.3 GiB 5.4 TiB
2.71 0.97 197 up osd.26
28 hdd 5.57169 1.00000 5.6 TiB 155 GiB 38 GiB 48 MiB 1.8 GiB 5.4 TiB
2.72 0.98 196 up osd.28
TOTAL 167 TiB 4.7 TiB 1.2 TiB 952 MiB 57 GiB 162 TiB
2.79
MIN/MAX VAR: 0.95/1.04 STDDEV: 0.06
Thanks for your help.
Regards,
Lars
{
"state": "activating+undersized+degraded+remapped+forced_backfill",
"snap_trimq": "[]",
"snap_trimq_len": 0,
"epoch": 33825,
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"backfill_targets": [
"7(0)",
"8(5)",
"9(2)",
"29(1)"
],
"acting_recovery_backfill": [
"2(5)",
"5(0)",
"7(0)",
"8(5)",
"9(2)",
"11(4)",
"25(2)",
"28(3)",
"29(1)"
],
"info": {
"pgid": "21.2e4s0",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30371'6388320",
"last_user_version": 6391413,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "31341'6391413",
"reported_seq": "7712809",
"reported_epoch": "33825",
"state": "activating+undersized+degraded+remapped+forced_backfill",
"last_fresh": "2019-06-19 07:32:55.470088",
"last_change": "2019-06-18 08:50:56.348526",
"last_active": "2019-06-17 16:37:59.365137",
"last_peered": "2019-06-17 16:37:57.226539",
"last_clean": "2019-06-17 16:37:16.346645",
"last_became_active": "2019-06-17 16:37:57.139223",
"last_became_peered": "2019-06-17 16:37:57.139223",
"last_unstale": "2019-06-19 07:32:55.470088",
"last_undegraded": "2019-06-17 16:38:13.010037",
"last_fullsized": "2019-06-17 16:38:12.996641",
"mapping_epoch": 31387,
"log_start": "30371'6388320",
"ondisk_log_start": "30371'6388320",
"created": 27405,
"last_epoch_clean": 31317,
"parent": "0.0",
"parent_split_bits": 10,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235",
"log_size": 3093,
"ondisk_log_size": 3093,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 801552730,
"num_objects": 346,
"num_object_clones": 66,
"num_object_copies": 2076,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 346,
"num_objects_misplaced": 1038,
"num_objects_unfound": 0,
"num_objects_dirty": 346,
"num_whiteouts": 2,
"num_read": 2927,
"num_read_kb": 91973,
"num_write": 5294,
"num_write_kb": 52404,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 602,
"num_bytes_recovered": 1566968602,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31388,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "2(5)",
"pgid": "21.2e4s5",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30371'6388320",
"last_user_version": 6391413,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "31341'6391413",
"reported_seq": "7710252",
"reported_epoch": "31341",
"state": "active+clean",
"last_fresh": "2019-06-17 16:30:47.917840",
"last_change": "2019-06-17 14:42:48.272287",
"last_active": "2019-06-17 16:30:47.917840",
"last_peered": "2019-06-17 16:30:47.917840",
"last_clean": "2019-06-17 16:30:47.917840",
"last_became_active": "2019-06-06 11:13:57.857566",
"last_became_peered": "2019-06-06 11:13:57.857566",
"last_unstale": "2019-06-17 16:30:47.917840",
"last_undegraded": "2019-06-17 16:30:47.917840",
"last_fullsized": "2019-06-17 16:30:47.917840",
"mapping_epoch": 31387,
"log_start": "30371'6388320",
"ondisk_log_start": "30371'6388320",
"created": 27405,
"last_epoch_clean": 31317,
"parent": "0.0",
"parent_split_bits": 10,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235",
"log_size": 3093,
"ondisk_log_size": 3093,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 801552730,
"num_objects": 346,
"num_object_clones": 66,
"num_object_copies": 2076,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 346,
"num_whiteouts": 2,
"num_read": 2927,
"num_read_kb": 91973,
"num_write": 5294,
"num_write_kb": 52404,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 602,
"num_bytes_recovered": 1566968602,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31379,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "3(1)",
"pgid": "21.2e4s1",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30371'6388320",
"last_user_version": 6391413,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "31341'6391413",
"reported_seq": "7710252",
"reported_epoch": "31341",
"state": "active+clean",
"last_fresh": "2019-06-17 16:30:47.917840",
"last_change": "2019-06-17 14:42:48.272287",
"last_active": "2019-06-17 16:30:47.917840",
"last_peered": "2019-06-17 16:30:47.917840",
"last_clean": "2019-06-17 16:30:47.917840",
"last_became_active": "2019-06-06 11:13:57.857566",
"last_became_peered": "2019-06-06 11:13:57.857566",
"last_unstale": "2019-06-17 16:30:47.917840",
"last_undegraded": "2019-06-17 16:30:47.917840",
"last_fullsized": "2019-06-17 16:30:47.917840",
"mapping_epoch": 31387,
"log_start": "30371'6388320",
"ondisk_log_start": "30371'6388320",
"created": 27405,
"last_epoch_clean": 31317,
"parent": "0.0",
"parent_split_bits": 10,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235",
"log_size": 3093,
"ondisk_log_size": 3093,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 801552730,
"num_objects": 346,
"num_object_clones": 66,
"num_object_copies": 2076,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 346,
"num_whiteouts": 2,
"num_read": 2927,
"num_read_kb": 91973,
"num_write": 5294,
"num_write_kb": 52404,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 602,
"num_bytes_recovered": 1566968602,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31379,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "7(0)",
"pgid": "21.2e4s0",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30372'6388413",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31387,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 346,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31370,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "8(5)",
"pgid": "21.2e4s5",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30372'6388413",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 346,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [],
"acting": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31388,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "9(2)",
"pgid": "21.2e4s2",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30372'6388413",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31387,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 346,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31360,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "11(4)",
"pgid": "21.2e4s4",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30371'6388320",
"last_user_version": 6391413,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "31341'6391413",
"reported_seq": "7710252",
"reported_epoch": "31341",
"state": "active+clean",
"last_fresh": "2019-06-17 16:30:47.917840",
"last_change": "2019-06-17 14:42:48.272287",
"last_active": "2019-06-17 16:30:47.917840",
"last_peered": "2019-06-17 16:30:47.917840",
"last_clean": "2019-06-17 16:30:47.917840",
"last_became_active": "2019-06-06 11:13:57.857566",
"last_became_peered": "2019-06-06 11:13:57.857566",
"last_unstale": "2019-06-17 16:30:47.917840",
"last_undegraded": "2019-06-17 16:30:47.917840",
"last_fullsized": "2019-06-17 16:30:47.917840",
"mapping_epoch": 31387,
"log_start": "30371'6388320",
"ondisk_log_start": "30371'6388320",
"created": 27405,
"last_epoch_clean": 31317,
"parent": "0.0",
"parent_split_bits": 10,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235",
"log_size": 3093,
"ondisk_log_size": 3093,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 801552730,
"num_objects": 346,
"num_object_clones": 66,
"num_object_copies": 2076,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 346,
"num_whiteouts": 2,
"num_read": 2927,
"num_read_kb": 91973,
"num_write": 5294,
"num_write_kb": 52404,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 602,
"num_bytes_recovered": 1566968602,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31379,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "17(5)",
"pgid": "21.2e4s5",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30372'6388413",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31370,
"last_interval_started": 31369,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31387,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 346,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31367,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "25(2)",
"pgid": "21.2e4s2",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30371'6388320",
"last_user_version": 6391413,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "31341'6391413",
"reported_seq": "7710252",
"reported_epoch": "31341",
"state": "active+clean",
"last_fresh": "2019-06-17 16:30:47.917840",
"last_change": "2019-06-17 14:42:48.272287",
"last_active": "2019-06-17 16:30:47.917840",
"last_peered": "2019-06-17 16:30:47.917840",
"last_clean": "2019-06-17 16:30:47.917840",
"last_became_active": "2019-06-06 11:13:57.857566",
"last_became_peered": "2019-06-06 11:13:57.857566",
"last_unstale": "2019-06-17 16:30:47.917840",
"last_undegraded": "2019-06-17 16:30:47.917840",
"last_fullsized": "2019-06-17 16:30:47.917840",
"mapping_epoch": 31387,
"log_start": "30371'6388320",
"ondisk_log_start": "30371'6388320",
"created": 27405,
"last_epoch_clean": 31317,
"parent": "0.0",
"parent_split_bits": 10,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235",
"log_size": 3093,
"ondisk_log_size": 3093,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 801552730,
"num_objects": 346,
"num_object_clones": 66,
"num_object_copies": 2076,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 346,
"num_whiteouts": 2,
"num_read": 2927,
"num_read_kb": 91973,
"num_write": 5294,
"num_write_kb": 52404,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 602,
"num_bytes_recovered": 1566968602,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31379,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "25(5)",
"pgid": "21.2e4s5",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30372'6388413",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31387,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 346,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31360,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "28(3)",
"pgid": "21.2e4s3",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30371'6388320",
"last_user_version": 6391413,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "31341'6391413",
"reported_seq": "7710252",
"reported_epoch": "31341",
"state": "active+clean",
"last_fresh": "2019-06-17 16:30:47.917840",
"last_change": "2019-06-17 14:42:48.272287",
"last_active": "2019-06-17 16:30:47.917840",
"last_peered": "2019-06-17 16:30:47.917840",
"last_clean": "2019-06-17 16:30:47.917840",
"last_became_active": "2019-06-06 11:13:57.857566",
"last_became_peered": "2019-06-06 11:13:57.857566",
"last_unstale": "2019-06-17 16:30:47.917840",
"last_undegraded": "2019-06-17 16:30:47.917840",
"last_fullsized": "2019-06-17 16:30:47.917840",
"mapping_epoch": 31387,
"log_start": "30371'6388320",
"ondisk_log_start": "30371'6388320",
"created": 27405,
"last_epoch_clean": 31317,
"parent": "0.0",
"parent_split_bits": 10,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235",
"log_size": 3093,
"ondisk_log_size": 3093,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 801552730,
"num_objects": 346,
"num_object_clones": 66,
"num_object_copies": 2076,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 346,
"num_whiteouts": 2,
"num_read": 2927,
"num_read_kb": 91973,
"num_write": 5294,
"num_write_kb": 52404,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 602,
"num_bytes_recovered": 1566968602,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31379,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "29(1)",
"pgid": "21.2e4s1",
"last_update": "31341'6391413",
"last_complete": "31341'6391413",
"log_tail": "30372'6388413",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 27405,
"epoch_pool_created": 19265,
"last_epoch_started": 31376,
"last_interval_started": 31375,
"last_epoch_clean": 31317,
"last_interval_clean": 31316,
"last_epoch_split": 27405,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31387,
"same_primary_since": 30362,
"last_scrub": "31341'6391383",
"last_scrub_stamp": "2019-06-17 14:42:48.272235",
"last_deep_scrub": "31339'6390969",
"last_deep_scrub_stamp": "2019-06-16 14:15:00.294302",
"last_clean_scrub_stamp": "2019-06-17 14:42:48.272235"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31387,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 346,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
29,
9,
28,
11,
8
],
"acting": [
5,
2147483647,
25,
28,
11,
2
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 5,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31370,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2019-06-17 16:38:12.996819",
"might_have_unfound": [
{
"osd": "3(1)",
"status": "not queried"
}
],
"recovery_progress": {
"backfill_targets": [
"7(0)",
"8(5)",
"9(2)",
"29(1)"
],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"recovery_ops": [],
"read_ops": []
}
},
"scrub": {
"scrubber.epoch_start": "31316",
"scrubber.active": false,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.max_end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2019-06-17 16:38:11.950533"
}
],
"agent_state": {}
}
{
"state": "activating+undersized+degraded+remapped+forced_backfill",
"snap_trimq": "[]",
"snap_trimq_len": 0,
"epoch": 33825,
"up": [
13,
21,
23
],
"acting": [
13,
21
],
"backfill_targets": [
"23"
],
"acting_recovery_backfill": [
"13",
"21",
"23"
],
"info": {
"pgid": "23.5",
"last_update": "31341'8502753",
"last_complete": "31341'8502753",
"log_tail": "31341'8499730",
"last_user_version": 8502753,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 29500,
"epoch_pool_created": 19802,
"last_epoch_started": 31370,
"last_interval_started": 31369,
"last_epoch_clean": 30356,
"last_interval_clean": 30355,
"last_epoch_split": 29500,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31379,
"same_primary_since": 31379,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054"
},
"stats": {
"version": "31341'8502753",
"reported_seq": "8542273",
"reported_epoch": "33825",
"state": "activating+undersized+degraded+remapped+forced_backfill",
"last_fresh": "2019-06-19 07:32:54.565678",
"last_change": "2019-06-18 08:51:04.822175",
"last_active": "2019-06-17 16:38:00.880590",
"last_peered": "2019-06-17 16:36:15.771563",
"last_clean": "2019-06-17 16:36:15.771563",
"last_became_active": "2019-05-29 11:26:09.615232",
"last_became_peered": "2019-05-29 11:26:09.615232",
"last_unstale": "2019-06-19 07:32:54.565678",
"last_undegraded": "2019-06-17 16:38:03.699889",
"last_fullsized": "2019-06-17 16:38:02.569918",
"mapping_epoch": 31379,
"log_start": "31341'8499730",
"ondisk_log_start": "31341'8499730",
"created": 29500,
"last_epoch_clean": 30356,
"parent": "0.0",
"parent_split_bits": 7,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054",
"log_size": 3023,
"ondisk_log_size": 3023,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 4194304,
"num_objects": 332,
"num_object_clones": 0,
"num_object_copies": 996,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 332,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 332,
"num_whiteouts": 0,
"num_read": 342644,
"num_read_kb": 7021,
"num_write": 1539734,
"num_write_kb": 9960774,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 331,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 6807414,
"num_omap_keys": 11631,
"num_objects_repaired": 0
},
"up": [
13,
21,
23
],
"acting": [
13,
21
],
"blocked_by": [],
"up_primary": 13,
"acting_primary": 13,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31381,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "4",
"pgid": "23.5",
"last_update": "31341'8502753",
"last_complete": "31341'8502753",
"log_tail": "31341'8499753",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 29500,
"epoch_pool_created": 19802,
"last_epoch_started": 31370,
"last_interval_started": 31369,
"last_epoch_clean": 30356,
"last_interval_clean": 30355,
"last_epoch_split": 29500,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31379,
"same_primary_since": 31379,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31379,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 332,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
13,
21,
23
],
"acting": [
13,
21
],
"blocked_by": [],
"up_primary": 13,
"acting_primary": 13,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31356,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "12",
"pgid": "23.5",
"last_update": "31341'8502753",
"last_complete": "31341'8502753",
"log_tail": "31341'8499753",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 29500,
"epoch_pool_created": 19802,
"last_epoch_started": 31370,
"last_interval_started": 31369,
"last_epoch_clean": 30356,
"last_interval_clean": 30355,
"last_epoch_split": 29500,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31379,
"same_primary_since": 31379,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31379,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 332,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
13,
21,
23
],
"acting": [
13,
21
],
"blocked_by": [],
"up_primary": 13,
"acting_primary": 13,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31354,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "15",
"pgid": "23.5",
"last_update": "31341'8502753",
"last_complete": "31341'8502753",
"log_tail": "31341'8499753",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 29500,
"epoch_pool_created": 19802,
"last_epoch_started": 31367,
"last_interval_started": 31366,
"last_epoch_clean": 30356,
"last_interval_clean": 30355,
"last_epoch_split": 29500,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31379,
"same_primary_since": 31379,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31379,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 332,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
13,
21,
23
],
"acting": [
13,
21
],
"blocked_by": [],
"up_primary": 13,
"acting_primary": 13,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31367,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "16",
"pgid": "23.5",
"last_update": "31341'8502753",
"last_complete": "31341'8502753",
"log_tail": "31341'8499730",
"last_user_version": 8502753,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 29500,
"epoch_pool_created": 19802,
"last_epoch_started": 31359,
"last_interval_started": 31358,
"last_epoch_clean": 30356,
"last_interval_clean": 30355,
"last_epoch_split": 29500,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31379,
"same_primary_since": 31379,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054"
},
"stats": {
"version": "31341'8502752",
"reported_seq": "8539811",
"reported_epoch": "31341",
"state": "active+clean",
"last_fresh": "2019-06-17 16:31:24.190596",
"last_change": "2019-06-16 21:55:27.972139",
"last_active": "2019-06-17 16:31:24.190596",
"last_peered": "2019-06-17 16:31:24.190596",
"last_clean": "2019-06-17 16:31:24.190596",
"last_became_active": "2019-05-29 11:26:09.615232",
"last_became_peered": "2019-05-29 11:26:09.615232",
"last_unstale": "2019-06-17 16:31:24.190596",
"last_undegraded": "2019-06-17 16:31:24.190596",
"last_fullsized": "2019-06-17 16:31:24.190596",
"mapping_epoch": 31379,
"log_start": "31341'8499730",
"ondisk_log_start": "31341'8499730",
"created": 29500,
"last_epoch_clean": 30356,
"parent": "0.0",
"parent_split_bits": 7,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054",
"log_size": 3022,
"ondisk_log_size": 3022,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 4194304,
"num_objects": 332,
"num_object_clones": 0,
"num_object_copies": 996,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 332,
"num_whiteouts": 0,
"num_read": 342644,
"num_read_kb": 7021,
"num_write": 1539734,
"num_write_kb": 9960774,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 331,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 6807414,
"num_omap_keys": 11631,
"num_objects_repaired": 0
},
"up": [
13,
21,
23
],
"acting": [
13,
21
],
"blocked_by": [],
"up_primary": 13,
"acting_primary": 13,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31359,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "19",
"pgid": "23.5",
"last_update": "31341'8502753",
"last_complete": "31341'8502753",
"log_tail": "31341'8499753",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 29500,
"epoch_pool_created": 19802,
"last_epoch_started": 31370,
"last_interval_started": 31369,
"last_epoch_clean": 30356,
"last_interval_clean": 30355,
"last_epoch_split": 29500,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31379,
"same_primary_since": 31379,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 31379,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 332,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
13,
21,
23
],
"acting": [
13,
21
],
"blocked_by": [],
"up_primary": 13,
"acting_primary": 13,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31370,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "21",
"pgid": "23.5",
"last_update": "31341'8502753",
"last_complete": "31341'8502753",
"log_tail": "31341'8499730",
"last_user_version": 8502753,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 29500,
"epoch_pool_created": 19802,
"last_epoch_started": 31370,
"last_interval_started": 31369,
"last_epoch_clean": 30356,
"last_interval_clean": 30355,
"last_epoch_split": 29500,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31379,
"same_primary_since": 31379,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054"
},
"stats": {
"version": "31341'8502753",
"reported_seq": "8539885",
"reported_epoch": "31378",
"state": "remapped+peering",
"last_fresh": "2019-06-17 16:37:59.374756",
"last_change": "2019-06-17 16:37:59.374756",
"last_active": "2019-06-17 16:37:59.374632",
"last_peered": "2019-06-17 16:37:57.302944",
"last_clean": "2019-06-17 16:31:24.190596",
"last_became_active": "2019-06-17 16:37:47.856895",
"last_became_peered": "2019-06-17 16:37:47.856895",
"last_unstale": "2019-06-17 16:37:59.374756",
"last_undegraded": "2019-06-17 16:37:59.374756",
"last_fullsized": "2019-06-17 16:37:59.374756",
"mapping_epoch": 31379,
"log_start": "31341'8499730",
"ondisk_log_start": "31341'8499730",
"created": 29500,
"last_epoch_clean": 30356,
"parent": "0.0",
"parent_split_bits": 7,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054",
"log_size": 3023,
"ondisk_log_size": 3023,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 4194304,
"num_objects": 332,
"num_object_clones": 0,
"num_object_copies": 996,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 332,
"num_whiteouts": 0,
"num_read": 342644,
"num_read_kb": 7021,
"num_write": 1539734,
"num_write_kb": 9960774,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 331,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 6807414,
"num_omap_keys": 11631,
"num_objects_repaired": 0
},
"up": [
13,
21,
23
],
"acting": [
13,
21
],
"blocked_by": [
12,
13,
23
],
"up_primary": 13,
"acting_primary": 13,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 31370,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "23",
"pgid": "23.5",
"last_update": "31341'8502753",
"last_complete": "31341'8502753",
"log_tail": "31341'8499753",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 29500,
"epoch_pool_created": 19802,
"last_epoch_started": 31370,
"last_interval_started": 31369,
"last_epoch_clean": 30356,
"last_interval_clean": 30355,
"last_epoch_split": 29500,
"last_epoch_marked_full": 0,
"same_up_since": 31378,
"same_interval_since": 31379,
"same_primary_since": 31379,
"last_scrub": "31339'8472062",
"last_scrub_stamp": "2019-06-16 21:55:27.972054",
"last_deep_scrub": "31335'8393499",
"last_deep_scrub_stamp": "2019-06-14 14:07:27.642470",
"last_clean_scrub_stamp": "2019-06-16 21:55:27.972054"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 332,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [],
"acting": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 31381,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2019-06-17 16:38:03.695681",
"might_have_unfound": [
{
"osd": "12",
"status": "not queried"
},
{
"osd": "16",
"status": "not queried"
},
{
"osd": "19",
"status": "not queried"
}
],
"recovery_progress": {
"backfill_targets": [
"23"
],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "30355",
"scrubber.active": false,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.max_end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2019-06-17 16:38:00.880625"
}
],
"agent_state": {}
}
_______________________________________________
ceph-users mailing list
[email protected]
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
