---------- Пересылаемое сообщение ---------
От: *Voloshanenko Igor* <[email protected]>
Дата: вторник, 18 августа 2015 г.
Тема: Repair inconsistent pgs..
Кому: Irek Fasikhov <[email protected]>
Some additional inforamtion (Tnx Irek for questions!)
Pool values:
root@test:~# ceph osd pool get cold-storage size
size: 3
root@test:~# ceph osd pool get cold-storage min_size
min_size: 2
Broken pgs dump
PG_1 #########
{
"state": "active+clean+inconsistent",
"snap_trimq": "[]",
"epoch": 17541,
"up": [
56,
10,
42
],
"acting": [
56,
10,
42
],
"actingbackfill": [
"10",
"42",
"56"
],
"info": {
"pgid": "2.c4",
"last_update": "17541'29153",
"last_complete": "17541'29153",
"log_tail": "16746'26095",
"last_user_version": 401173,
"last_backfill": "MAX",
"purged_snaps":
"[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17531,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17530,
"same_primary_since": 17530,
"last_scrub": "17541'29114",
"last_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_deep_scrub": "17541'29114",
"last_deep_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731"
},
"stats": {
"version": "17541'29153",
"reported_seq": "21552",
"reported_epoch": "17541",
"state": "active+clean+inconsistent",
"last_fresh": "2015-08-18 07:48:37.667036",
"last_change": "2015-08-18 07:37:04.568541",
"last_active": "2015-08-18 07:48:37.667036",
"last_peered": "2015-08-18 07:48:37.667036",
"last_clean": "2015-08-18 07:48:37.667036",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 07:48:37.667036",
"last_undegraded": "2015-08-18 07:48:37.667036",
"last_fullsized": "2015-08-18 07:48:37.667036",
"mapping_epoch": 17140,
"log_start": "16746'26095",
"ondisk_log_start": "16746'26095",
"created": 98,
"last_epoch_clean": 17541,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17541'29114",
"last_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_deep_scrub": "17541'29114",
"last_deep_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 2236608990,
"num_objects": 307,
"num_object_clones": 7,
"num_object_copies": 921,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 307,
"num_whiteouts": 0,
"num_read": 15694,
"num_read_kb": 401354,
"num_write": 55720,
"num_write_kb": 2539827,
"num_scrub_errors": 1,
"num_shallow_scrub_errors": 1,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1842,
"num_bytes_recovered": 13419653940,
"num_keys_recovered": 36,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
10,
42
],
"acting": [
56,
10,
42
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17531,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
},
"peer_info": [
{
"peer": "10",
"pgid": "2.c4",
"last_update": "17541'29153",
"last_complete": "17541'29153",
"log_tail": "16746'25703",
"last_user_version": 400914,
"last_backfill": "MAX",
"purged_snaps":
"[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17531,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17530,
"same_primary_since": 17530,
"last_scrub": "17541'29114",
"last_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_deep_scrub": "17541'29114",
"last_deep_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731"
},
"stats": {
"version": "17527'28894",
"reported_seq": "21277",
"reported_epoch": "17529",
"state": "active+remapped+inconsistent+backfilling",
"last_fresh": "2015-08-18 05:07:02.140274",
"last_change": "2015-08-18 05:05:47.908928",
"last_active": "2015-08-18 05:07:02.140274",
"last_peered": "2015-08-18 05:07:02.140274",
"last_clean": "2015-08-18 02:52:55.811384",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 05:07:02.140274",
"last_undegraded": "2015-08-18 05:07:02.140274",
"last_fullsized": "2015-08-18 05:07:02.140274",
"mapping_epoch": 17140,
"log_start": "16746'25642",
"ondisk_log_start": "16746'25642",
"created": 98,
"last_epoch_clean": 17117,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17129'28702",
"last_scrub_stamp": "2015-08-18 02:52:55.810489",
"last_deep_scrub": "17129'28702",
"last_deep_scrub_stamp": "2015-08-18 02:52:55.810489",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731",
"log_size": 3252,
"ondisk_log_size": 3252,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 2236608990,
"num_objects": 307,
"num_object_clones": 7,
"num_object_copies": 1842,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 924,
"num_objects_unfound": 0,
"num_objects_dirty": 307,
"num_whiteouts": 0,
"num_read": 15475,
"num_read_kb": 401354,
"num_write": 55202,
"num_write_kb": 2538487,
"num_scrub_errors": 1,
"num_shallow_scrub_errors": 1,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1842,
"num_bytes_recovered": 13419653940,
"num_keys_recovered": 36,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
10,
42
],
"acting": [
56,
10,
42
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17531,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
},
{
"peer": "42",
"pgid": "2.c4",
"last_update": "17541'29153",
"last_complete": "17541'29153",
"log_tail": "16746'25703",
"last_user_version": 400914,
"last_backfill": "MAX",
"purged_snaps":
"[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17531,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17530,
"same_primary_since": 17530,
"last_scrub": "17541'29114",
"last_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_deep_scrub": "17541'29114",
"last_deep_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731"
},
"stats": {
"version": "17527'28894",
"reported_seq": "21277",
"reported_epoch": "17529",
"state": "active+remapped+inconsistent+backfilling",
"last_fresh": "2015-08-18 05:07:02.140274",
"last_change": "2015-08-18 05:05:47.908928",
"last_active": "2015-08-18 05:07:02.140274",
"last_peered": "2015-08-18 05:07:02.140274",
"last_clean": "2015-08-18 02:52:55.811384",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 05:07:02.140274",
"last_undegraded": "2015-08-18 05:07:02.140274",
"last_fullsized": "2015-08-18 05:07:02.140274",
"mapping_epoch": 17140,
"log_start": "16746'25642",
"ondisk_log_start": "16746'25642",
"created": 98,
"last_epoch_clean": 17117,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17129'28702",
"last_scrub_stamp": "2015-08-18 02:52:55.810489",
"last_deep_scrub": "17129'28702",
"last_deep_scrub_stamp": "2015-08-18 02:52:55.810489",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731",
"log_size": 3252,
"ondisk_log_size": 3252,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 2236608990,
"num_objects": 307,
"num_object_clones": 7,
"num_object_copies": 1842,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 924,
"num_objects_unfound": 0,
"num_objects_dirty": 307,
"num_whiteouts": 0,
"num_read": 15475,
"num_read_kb": 401354,
"num_write": 55202,
"num_write_kb": 2538487,
"num_scrub_errors": 1,
"num_shallow_scrub_errors": 1,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1842,
"num_bytes_recovered": 13419653940,
"num_keys_recovered": 36,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
10,
42
],
"acting": [
56,
10,
42
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17531,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2015-08-18 05:07:04.026226",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "0\/\/0\/\/-1",
"backfill_info": {
"begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "17530",
"scrubber.active": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2015-08-18 05:07:02.925534"
}
],
"agent_state": {}
}
PG_2 ############
{
"state": "active+clean+inconsistent",
"snap_trimq": "[]",
"epoch": 17541,
"up": [
56,
15,
29
],
"acting": [
56,
15,
29
],
"actingbackfill": [
"15",
"29",
"56"
],
"info": {
"pgid": "2.490",
"last_update": "17541'38040",
"last_complete": "17541'38040",
"log_tail": "16745'34960",
"last_user_version": 155503,
"last_backfill": "MAX",
"purged_snaps":
"[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17143,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17142,
"same_primary_since": 17142,
"last_scrub": "17541'38035",
"last_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_deep_scrub": "17541'38035",
"last_deep_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297"
},
"stats": {
"version": "17541'38040",
"reported_seq": "29406",
"reported_epoch": "17541",
"state": "active+clean+inconsistent",
"last_fresh": "2015-08-18 07:45:42.152254",
"last_change": "2015-08-18 07:37:13.983710",
"last_active": "2015-08-18 07:45:42.152254",
"last_peered": "2015-08-18 07:45:42.152254",
"last_clean": "2015-08-18 07:45:42.152254",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 07:45:42.152254",
"last_undegraded": "2015-08-18 07:45:42.152254",
"last_fullsized": "2015-08-18 07:45:42.152254",
"mapping_epoch": 17139,
"log_start": "16745'34960",
"ondisk_log_start": "16745'34960",
"created": 98,
"last_epoch_clean": 17541,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17541'38035",
"last_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_deep_scrub": "17541'38035",
"last_deep_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297",
"log_size": 3080,
"ondisk_log_size": 3080,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1308410368,
"num_objects": 318,
"num_object_clones": 4,
"num_object_copies": 954,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 318,
"num_whiteouts": 0,
"num_read": 19307,
"num_read_kb": 300579,
"num_write": 73378,
"num_write_kb": 2887251,
"num_scrub_errors": 17,
"num_shallow_scrub_errors": 17,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2322,
"num_bytes_recovered": 9568430592,
"num_keys_recovered": 45,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
15,
29
],
"acting": [
56,
15,
29
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17143,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
},
"peer_info": [
{
"peer": "15",
"pgid": "2.490",
"last_update": "17541'38040",
"last_complete": "17541'38040",
"log_tail": "16745'34855",
"last_user_version": 155325,
"last_backfill": "MAX",
"purged_snaps":
"[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17143,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17142,
"same_primary_since": 17142,
"last_scrub": "17541'38035",
"last_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_deep_scrub": "17541'38035",
"last_deep_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297"
},
"stats": {
"version": "17057'37860",
"reported_seq": "29201",
"reported_epoch": "17142",
"state": "active+remapped+inconsistent",
"last_fresh": "2015-08-18 02:54:29.982572",
"last_change": "2015-08-18 02:54:29.982572",
"last_active": "2015-08-18 02:54:29.982572",
"last_peered": "2015-08-18 02:54:29.982572",
"last_clean": "2015-08-18 02:45:44.150922",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 02:54:29.982572",
"last_undegraded": "2015-08-18 02:54:29.982572",
"last_fullsized": "2015-08-18 02:54:29.982572",
"mapping_epoch": 17139,
"log_start": "16745'34855",
"ondisk_log_start": "16745'34855",
"created": 98,
"last_epoch_clean": 17141,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17057'37860",
"last_scrub_stamp": "2015-08-18 02:47:56.695560",
"last_deep_scrub": "17057'37860",
"last_deep_scrub_stamp": "2015-08-18 02:47:56.695560",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297",
"log_size": 3005,
"ondisk_log_size": 3005,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1308410368,
"num_objects": 318,
"num_object_clones": 4,
"num_object_copies": 1590,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 636,
"num_objects_unfound": 0,
"num_objects_dirty": 318,
"num_whiteouts": 0,
"num_read": 19124,
"num_read_kb": 299799,
"num_write": 73022,
"num_write_kb": 2883469,
"num_scrub_errors": 17,
"num_shallow_scrub_errors": 17,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2322,
"num_bytes_recovered": 9568430592,
"num_keys_recovered": 45,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
15,
29
],
"acting": [
56,
15,
29
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17143,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
},
{
"peer": "29",
"pgid": "2.490",
"last_update": "17541'38040",
"last_complete": "17541'38040",
"log_tail": "16745'34860",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps":
"[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17143,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17142,
"same_primary_since": 17142,
"last_scrub": "17541'38035",
"last_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_deep_scrub": "17541'38035",
"last_deep_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297"
},
"stats": {
"version": "17057'37860",
"reported_seq": "29198",
"reported_epoch": "17141",
"state": "active+remapped+inconsistent+backfilling",
"last_fresh": "2015-08-18 02:54:29.979923",
"last_change": "2015-08-18 02:53:45.383431",
"last_active": "2015-08-18 02:54:29.979923",
"last_peered": "2015-08-18 02:54:29.979923",
"last_clean": "2015-08-18 02:45:44.150922",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 02:54:29.979923",
"last_undegraded": "2015-08-18 02:54:29.979923",
"last_fullsized": "2015-08-18 02:54:29.979923",
"mapping_epoch": 17139,
"log_start": "16745'34855",
"ondisk_log_start": "16745'34855",
"created": 98,
"last_epoch_clean": 17079,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17057'37860",
"last_scrub_stamp": "2015-08-18 02:47:56.695560",
"last_deep_scrub": "17057'37860",
"last_deep_scrub_stamp": "2015-08-18 02:47:56.695560",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297",
"log_size": 3005,
"ondisk_log_size": 3005,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1308410368,
"num_objects": 318,
"num_object_clones": 4,
"num_object_copies": 1590,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 638,
"num_objects_unfound": 0,
"num_objects_dirty": 318,
"num_whiteouts": 0,
"num_read": 19124,
"num_read_kb": 299799,
"num_write": 73022,
"num_write_kb": 2883469,
"num_scrub_errors": 17,
"num_shallow_scrub_errors": 17,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2322,
"num_bytes_recovered": 9568430592,
"num_keys_recovered": 45,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
15,
29
],
"acting": [
56,
15,
29
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17143,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2015-08-18 02:54:32.017624",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "0\/\/0\/\/-1",
"backfill_info": {
"begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "17142",
"scrubber.active": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2015-08-18 02:54:30.940793"
}
],
"agent_state": {}
}
2015-08-18 8:40 GMT+03:00 Irek Fasikhov <[email protected]
<javascript:_e(%7B%7D,'cvml','[email protected]');>>:
> Да просто recovery и repair немного разные вещи ;).
> Ну да ладно. не поняли просто друг друга.
> У тебя чему равен параметр min_size на пуле? и так же покажи вывод ceph pg
> $PG query
>
> С уважением, Фасихов Ирек Нургаязович
> Моб.: +79229045757
>
> 2015-08-18 8:36 GMT+03:00 Voloshanenko Igor <[email protected]
> <javascript:_e(%7B%7D,'cvml','[email protected]');>>:
>
>> Hi Irek, Please read careful )))
>> You proposal was the first, i try to do... That's why i asked about
>> help... (
>>
>> 2015-08-18 8:34 GMT+03:00 Irek Fasikhov <[email protected]
>> <javascript:_e(%7B%7D,'cvml','[email protected]');>>:
>>
>>> Hi, Igor.
>>>
>>> You need to repair the PG.
>>>
>>> for i in `ceph pg dump| grep inconsistent | grep -v
>>> 'inconsistent+repair' | awk {'print$1'}`;do ceph pg repair $i;done
>>>
>>> С уважением, Фасихов Ирек Нургаязович
>>> Моб.: +79229045757
>>>
>>> 2015-08-18 8:27 GMT+03:00 Voloshanenko Igor <[email protected]
>>> <javascript:_e(%7B%7D,'cvml','[email protected]');>>:
>>>
>>>> Hi all, at our production cluster, due high rebalancing ((( we have 2
>>>> pgs in inconsistent state...
>>>>
>>>> root@temp:~# ceph health detail | grep inc
>>>> HEALTH_ERR 2 pgs inconsistent; 18 scrub errors
>>>> pg 2.490 is active+clean+inconsistent, acting [56,15,29]
>>>> pg 2.c4 is active+clean+inconsistent, acting [56,10,42]
>>>>
>>>> From OSD logs, after recovery attempt:
>>>>
>>>> root@test:~# ceph pg dump | grep -i incons | cut -f 1 | while read i;
>>>> do ceph pg repair ${i} ; done
>>>> dumped all in format plain
>>>> instructing pg 2.490 on osd.56 to repair
>>>> instructing pg 2.c4 on osd.56 to repair
>>>>
>>>> /var/log/ceph/ceph-osd.56.log:51:2015-08-18 07:26:37.035910
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490
>>>> f5759490/rbd_data.1631755377d7e.00000000000004da/head//2 expected clone
>>>> 90c59490/rbd_data.eb486436f2beb.0000000000007a65/141//2
>>>> /var/log/ceph/ceph-osd.56.log:52:2015-08-18 07:26:37.035960
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490
>>>> fee49490/rbd_data.12483d3ba0794b.000000000000522f/head//2 expected clone
>>>> f5759490/rbd_data.1631755377d7e.00000000000004da/141//2
>>>> /var/log/ceph/ceph-osd.56.log:53:2015-08-18 07:26:37.036133
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490
>>>> a9b39490/rbd_data.12483d3ba0794b.00000000000037b3/head//2 expected clone
>>>> fee49490/rbd_data.12483d3ba0794b.000000000000522f/141//2
>>>> /var/log/ceph/ceph-osd.56.log:54:2015-08-18 07:26:37.036243
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490
>>>> bac19490/rbd_data.1238e82ae8944a.000000000000032e/head//2 expected clone
>>>> a9b39490/rbd_data.12483d3ba0794b.00000000000037b3/141//2
>>>> /var/log/ceph/ceph-osd.56.log:55:2015-08-18 07:26:37.036289
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490
>>>> 98519490/rbd_data.123e9c2ae8944a.0000000000000807/head//2 expected clone
>>>> bac19490/rbd_data.1238e82ae8944a.000000000000032e/141//2
>>>> /var/log/ceph/ceph-osd.56.log:56:2015-08-18 07:26:37.036314
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490
>>>> c3c09490/rbd_data.1238e82ae8944a.0000000000000c2b/head//2 expected clone
>>>> 98519490/rbd_data.123e9c2ae8944a.0000000000000807/141//2
>>>> /var/log/ceph/ceph-osd.56.log:57:2015-08-18 07:26:37.036363
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490
>>>> 28809490/rbd_data.edea7460fe42b.00000000000001d9/head//2 expected clone
>>>> c3c09490/rbd_data.1238e82ae8944a.0000000000000c2b/141//2
>>>> /var/log/ceph/ceph-osd.56.log:58:2015-08-18 07:26:37.036432
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490
>>>> e1509490/rbd_data.1423897545e146.00000000000009a6/head//2 expected clone
>>>> 28809490/rbd_data.edea7460fe42b.00000000000001d9/141//2
>>>> /var/log/ceph/ceph-osd.56.log:59:2015-08-18 07:26:38.548765
>>>> 7f94663b3700 -1 log_channel(cluster) log [ERR] : 2.490 deep-scrub 17 errors
>>>>
>>>> So, how i can solve "expected clone" situation by hand?
>>>> Thank in advance!
>>>>
>>>>
>>>>
>>>> _______________________________________________
>>>> ceph-users mailing list
>>>> [email protected]
>>>> <javascript:_e(%7B%7D,'cvml','[email protected]');>
>>>> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>>>>
>>>>
>>>
>>
>
_______________________________________________
ceph-users mailing list
[email protected]
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
