Here is the output of ceph pg query for one of hte
active+clean+inconsistent PGs:
{
"state": "active+clean+inconsistent",
"snap_trimq": "[]",
"epoch": 342982,
"up": [
319,
90,
51
],
"acting": [
319,
90,
51
],
"actingbackfill": [
"51",
"90",
"319"
],
"info": {
"pgid": "6.92c",
"last_update": "342982'41304",
"last_complete": "342982'41304",
"log_tail": "342980'38259",
"last_user_version": 41304,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 262553,
"last_epoch_started": 342598,
"last_epoch_clean": 342613,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 342596,
"same_interval_since": 342597,
"same_primary_since": 342597,
"last_scrub": "342982'41177",
"last_scrub_stamp": "2017-01-02 18:19:48.081750",
"last_deep_scrub": "342965'37465",
"last_deep_scrub_stamp": "2016-12-20 16:31:06.438823",
"last_clean_scrub_stamp": "2016-12-11 12:51:19.258816"
},
"stats": {
"version": "342982'41304",
"reported_seq": "43600",
"reported_epoch": "342982",
"state": "active+clean+inconsistent",
"last_fresh": "2017-01-03 15:27:15.075176",
"last_change": "2017-01-02 18:19:48.081806",
"last_active": "2017-01-03 15:27:15.075176",
"last_peered": "2017-01-03 15:27:15.075176",
"last_clean": "2017-01-03 15:27:15.075176",
"last_became_active": "2016-11-01 16:21:23.328639",
"last_became_peered": "2016-11-01 16:21:23.328639",
"last_unstale": "2017-01-03 15:27:15.075176",
"last_undegraded": "2017-01-03 15:27:15.075176",
"last_fullsized": "2017-01-03 15:27:15.075176",
"mapping_epoch": 342596,
"log_start": "342980'38259",
"ondisk_log_start": "342980'38259",
"created": 262553,
"last_epoch_clean": 342613,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "342982'41177",
"last_scrub_stamp": "2017-01-02 18:19:48.081750",
"last_deep_scrub": "342965'37465",
"last_deep_scrub_stamp": "2016-12-20 16:31:06.438823",
"last_clean_scrub_stamp": "2016-12-11 12:51:19.258816",
"log_size": 3045,
"ondisk_log_size": 3045,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": true,
"stat_sum": {
"num_bytes": 16929346269,
"num_objects": 4881,
"num_object_clones": 0,
"num_object_copies": 14643,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 4881,
"num_whiteouts": 0,
"num_read": 7592,
"num_read_kb": 19593996,
"num_write": 42541,
"num_write_kb": 47306915,
"num_scrub_errors": 1,
"num_shallow_scrub_errors": 1,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 5807,
"num_bytes_recovered": 22691211916,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
319,
90,
51
],
"acting": [
319,
90,
51
],
"blocked_by": [],
"up_primary": 319,
"acting_primary": 319
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 342598,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "51",
"pgid": "6.92c",
"last_update": "342982'41304",
"last_complete": "342982'41304",
"log_tail": "341563'12014",
"last_user_version": 15033,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 262553,
"last_epoch_started": 342598,
"last_epoch_clean": 342613,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 342596,
"same_interval_since": 342597,
"same_primary_since": 342597,
"last_scrub": "342982'41177",
"last_scrub_stamp": "2017-01-02 18:19:48.081750",
"last_deep_scrub": "342965'37465",
"last_deep_scrub_stamp": "2016-12-20 16:31:06.438823",
"last_clean_scrub_stamp": "2016-12-11 12:51:19.258816"
},
"stats": {
"version": "342541'15032",
"reported_seq": "21472",
"reported_epoch": "342597",
"state": "active+undersized+degraded",
"last_fresh": "2016-11-01 16:05:44.991004",
"last_change": "2016-11-01 16:05:44.990630",
"last_active": "2016-11-01 16:05:44.991004",
"last_peered": "2016-11-01 16:05:44.991004",
"last_clean": "2016-11-01 15:26:23.393984",
"last_became_active": "2016-11-01 16:05:44.990630",
"last_became_peered": "2016-11-01 16:05:44.990630",
"last_unstale": "2016-11-01 16:05:44.991004",
"last_undegraded": "2016-11-01 16:05:44.021269",
"last_fullsized": "2016-11-01 16:05:44.021269",
"mapping_epoch": 342596,
"log_start": "341563'12014",
"ondisk_log_start": "341563'12014",
"created": 262553,
"last_epoch_clean": 342587,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "342266'14514",
"last_scrub_stamp": "2016-10-28 16:41:06.563820",
"last_deep_scrub": "342266'14514",
"last_deep_scrub_stamp": "2016-10-28 16:41:06.563820",
"last_clean_scrub_stamp": "2016-10-28 16:41:06.563820",
"log_size": 3018,
"ondisk_log_size": 3018,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": true,
"stat_sum": {
"num_bytes": 12528581359,
"num_objects": 3562,
"num_object_clones": 0,
"num_object_copies": 10683,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 3561,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 3562,
"num_whiteouts": 0,
"num_read": 3678,
"num_read_kb": 10197642,
"num_write": 15656,
"num_write_kb": 19564203,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 5806,
"num_bytes_recovered": 22687335556,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
319,
90,
51
],
"acting": [
319,
90,
51
],
"blocked_by": [],
"up_primary": 319,
"acting_primary": 319
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 342598,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "90",
"pgid": "6.92c",
"last_update": "342982'41304",
"last_complete": "342982'41304",
"log_tail": "341563'12014",
"last_user_version": 15033,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 262553,
"last_epoch_started": 342598,
"last_epoch_clean": 342613,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 342596,
"same_interval_since": 342597,
"same_primary_since": 342597,
"last_scrub": "342982'41177",
"last_scrub_stamp": "2017-01-02 18:19:48.081750",
"last_deep_scrub": "342965'37465",
"last_deep_scrub_stamp": "2016-12-20 16:31:06.438823",
"last_clean_scrub_stamp": "2016-12-11 12:51:19.258816"
},
"stats": {
"version": "342589'15033",
"reported_seq": "21478",
"reported_epoch": "342596",
"state": "remapped+peering",
"last_fresh": "2016-11-01 16:21:20.584113",
"last_change": "2016-11-01 16:21:20.295685",
"last_active": "2016-11-01 16:14:02.694748",
"last_peered": "2016-11-01 16:14:02.694748",
"last_clean": "2016-11-01 15:26:23.393984",
"last_became_active": "2016-11-01 16:05:44.990630",
"last_became_peered": "2016-11-01 16:05:44.990630",
"last_unstale": "2016-11-01 16:21:20.584113",
"last_undegraded": "2016-11-01 16:21:20.584113",
"last_fullsized": "2016-11-01 16:21:20.584113",
"mapping_epoch": 342596,
"log_start": "341563'12014",
"ondisk_log_start": "341563'12014",
"created": 262553,
"last_epoch_clean": 342587,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "342266'14514",
"last_scrub_stamp": "2016-10-28 16:41:06.563820",
"last_deep_scrub": "342266'14514",
"last_deep_scrub_stamp": "2016-10-28 16:41:06.563820",
"last_clean_scrub_stamp": "2016-10-28 16:41:06.563820",
"log_size": 3019,
"ondisk_log_size": 3019,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": true,
"stat_sum": {
"num_bytes": 12528581359,
"num_objects": 3562,
"num_object_clones": 0,
"num_object_copies": 10686,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 3562,
"num_whiteouts": 0,
"num_read": 3678,
"num_read_kb": 10197642,
"num_write": 15656,
"num_write_kb": 19564203,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 5806,
"num_bytes_recovered": 22687335556,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
319,
90,
51
],
"acting": [
319,
90,
51
],
"blocked_by": [],
"up_primary": 319,
"acting_primary": 319
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 342598,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2016-11-01 16:21:23.007072",
"might_have_unfound": [
{
"osd": "51",
"status": "already probed"
},
{
"osd": "90",
"status": "already probed"
}
],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "342597",
"scrubber.active": 0,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.seed": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2016-11-01 16:21:21.763033"
}
],
"agent_state": {}
}
Andras
On 12/23/2016 01:27 AM, Shinobu Kinjo wrote:
Would you be able to execute ``ceph pg ${PG ID} query`` against that
particular PG?
On Wed, Dec 21, 2016 at 11:44 PM, Andras Pataki
<apat...@simonsfoundation.org> wrote:
Yes, size = 3, and I have checked that all three replicas are the same zero
length object on the disk. I think some metadata info is mismatching what
the OSD log refers to as "object info size". But I'm not sure what to do
about it. pg repair does not fix it. In fact, the file this object
corresponds to in CephFS is shorter so this chunk shouldn't even exist I
think (details are in the original email). Although I may be understanding
the situation wrong ...
Andras
On 12/21/2016 07:17 AM, Mehmet wrote:
Hi Andras,
Iam not the experienced User but i guess you could have a look on this
object on each related osd for the pg, compare them and delete the Different
object. I assume you have size = 3.
Then again pg repair.
But be carefull iirc the replica will be recovered from the primary pg.
Hth
Am 20. Dezember 2016 22:39:44 MEZ, schrieb Andras Pataki
<apat...@simonsfoundation.org>:
Hi cephers,
Any ideas on how to proceed on the inconsistencies below? At the moment
our ceph setup has 5 of these - in all cases it seems like some zero length
objects that match across the three replicas, but do not match the object
info size. I tried running pg repair on one of them, but it didn't repair
the problem:
2016-12-20 16:24:40.870307 7f3e1a4b1700 0 log_channel(cluster) log [INF]
: 6.92c repair starts
2016-12-20 16:27:06.183186 7f3e1a4b1700 -1 log_channel(cluster) log [ERR]
: repair 6.92c 6:34932257:::1000187bbb5.00000009:head on disk size (0) does
not match object info size (3014656) adjusted for ondisk to (3014656)
2016-12-20 16:27:35.885496 7f3e17cac700 -1 log_channel(cluster) log [ERR]
: 6.92c repair 1 errors, 0 fixed
Any help/hints would be appreciated.
Thanks,
Andras
On 12/15/2016 10:13 AM, Andras Pataki wrote:
Hi everyone,
Yesterday scrubbing turned up an inconsistency in one of our placement
groups. We are running ceph 10.2.3, using CephFS and RBD for some VM
images.
[root@hyperv017 ~]# ceph -s
cluster d7b33135-0940-4e48-8aa6-1d2026597c2f
health HEALTH_ERR
1 pgs inconsistent
1 scrub errors
noout flag(s) set
monmap e15: 3 mons at
{hyperv029=10.4.36.179:6789/0,hyperv030=10.4.36.180:6789/0,hyperv031=10.4.36.181:6789/0}
election epoch 27192, quorum 0,1,2
hyperv029,hyperv030,hyperv031
fsmap e17181: 1/1/1 up {0=hyperv029=up:active}, 2 up:standby
osdmap e342930: 385 osds: 385 up, 385 in
flags noout
pgmap v37580512: 34816 pgs, 5 pools, 673 TB data, 198 Mobjects
1583 TB used, 840 TB / 2423 TB avail
34809 active+clean
4 active+clean+scrubbing+deep
2 active+clean+scrubbing
1 active+clean+inconsistent
client io 87543 kB/s rd, 671 MB/s wr, 23 op/s rd, 2846 op/s wr
# ceph pg dump | grep inconsistent
6.13f1 4692 0 0 0 0 16057314767 3087 3087
active+clean+inconsistent 2016-12-14 16:49:48.391572 342929'41011
342929:43966 [158,215,364] 158 [158,215,364] 158 342928'40540
2016-12-14 16:49:48.391511 342928'40540 2016-12-14 16:49:48.391511
I tried a couple of other deep scrubs on pg 6.13f1 but got repeated
errors. In the OSD logs:
2016-12-14 16:48:07.733291 7f3b56e3a700 -1 log_channel(cluster) log [ERR]
: deep-scrub 6.13f1 6:8fc91b77:::1000187bb70.00000009:head on disk size (0)
does not match object info size (1835008) adjusted for ondisk to (1835008)
I looked at the objects on the 3 OSD's on their respective hosts and they
are the same, zero length files:
# cd ~ceph/osd/ceph-158/current/6.13f1_head
# find . -name *1000187bb70* -ls
669738 0 -rw-r--r-- 1 ceph ceph 0 Dec 13 17:00
./DIR_1/DIR_F/DIR_3/DIR_9/DIR_8/1000187bb70.00000009__head_EED893F1__6
# cd ~ceph/osd/ceph-215/current/6.13f1_head
# find . -name *1000187bb70* -ls
539815647 0 -rw-r--r-- 1 ceph ceph 0 Dec 13 17:00
./DIR_1/DIR_F/DIR_3/DIR_9/DIR_8/1000187bb70.00000009__head_EED893F1__6
# cd ~ceph/osd/ceph-364/current/6.13f1_head
# find . -name *1000187bb70* -ls
1881432215 0 -rw-r--r-- 1 ceph ceph 0 Dec 13 17:00
./DIR_1/DIR_F/DIR_3/DIR_9/DIR_8/1000187bb70.00000009__head_EED893F1__6
At the time of the write, there wasn't anything unusual going on as far as
I can tell (no hardware/network issues, all processes were up, etc).
This pool is a CephFS data pool, and the corresponding file (inode hex
1000187bb70, decimal 1099537300336) looks like this:
# ls -li chr4.tags.tsv
1099537300336 -rw-r--r-- 1 xichen xichen 14469915 Dec 13 17:01
chr4.tags.tsv
Reading the file is also ok (no errors, right number of bytes):
# cat chr4.tags.tsv > /dev/null
# wc chr4.tags.tsv
592251 2961255 14469915 chr4.tags.tsv
We are using the standard 4MB block size for CephFS, and if I interpret
this right, this is the 9th chunk, so there shouldn't be any data (or even a
9th chunk), since the file is only 14MB. Should I run pg repair on this?
Any ideas on how this could come about? Any other recommendations?
Thanks,
Andras
apat...@apataki.net
________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
