Hi Team,
In one of our ceph cluster we observe that there are many slow IOPS in all
our OSD's and most of the latency is happening between two set of
operations which are shown below.
{
"time": "2019-11-12 08:29:58.128669",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:08.484235",
"event": "commit_sent"
Is there any way to know what is causing this issue and rectify this
problem? As you can see in the above output of historical_slow_ops there is
a 10 second delay between the two operations which is impacting the
performance. Attaching one of the OSD historic slow ops output to this mail
for reference.
{
"num to keep": 20,
"threshold to keep": 10,
"Ops": [
{
"description": "osd_op(client.71027356.0:194654 1.7s0
1:e005f8bb:::%2fc20%2fvx0000039%2fpot%2fsridhar_106%2fhls%2fh_63538bf124417ce82e34317a6c95097a%2fvar2493000%2fseg20271245_w1573279521.ts:head
[delete] snapc 0=[] ondisk+write+peerstat_old+known_if_redirected e30337)",
"initiated_at": "2019-11-12 08:29:50.410364",
"age": 77561.712867374998,
"duration": 12.158651442,
"type_data": {
"flag_point": "commit sent; apply or cleanup",
"client_info": {
"client": "client.71027356",
"client_addr": "10.50.62.163:0/478395830",
"tid": 194654
},
"events": [
{
"time": "2019-11-12 08:29:50.410364",
"event": "initiated"
},
{
"time": "2019-11-12 08:29:50.410364",
"event": "header_read"
},
{
"time": "2019-11-12 08:29:50.410361",
"event": "throttled"
},
{
"time": "2019-11-12 08:29:50.410368",
"event": "all_read"
},
{
"time": "2019-11-12 08:29:50.410369",
"event": "dispatched"
},
{
"time": "2019-11-12 08:29:50.410374",
"event": "queued_for_pg"
},
{
"time": "2019-11-12 08:29:50.410392",
"event": "reached_pg"
},
{
"time": "2019-11-12 08:29:50.410562",
"event": "started"
},
{
"time": "2019-11-12 08:29:50.410684",
"event": "sub_op_started"
},
{
"time": "2019-11-12 08:29:50.419974",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:02.568999",
"event": "commit_sent"
},
{
"time": "2019-11-12 08:30:02.569015",
"event": "done"
}
]
}
},
{
"description": "osd_op(client.71027356.0:195247 1.7s0
1:e0013cdb:::%2fc20%2fvx0000039%2fpot%2fsridhar_84%2fhls%2fh_598d9980480259996ca22bd53af09f02%2fvar370000%2fseg20315902_w1573547390.ts:head
[writefull 0~220148,setxattr mode (7),setxattr uid (5),setxattr gid
(5),setxattr size (7),setxattr mtime (11),setxattr xattr (2),setxattr
meal_flags (3),setxattr expire (7),setxattr md5 (33)] snapc 0=[]
ondisk+write+known_if_redirected e30337)",
"initiated_at": "2019-11-12 08:29:56.741549",
"age": 77555.381682584004,
"duration": 11.742638746000001,
"type_data": {
"flag_point": "commit sent; apply or cleanup",
"client_info": {
"client": "client.71027356",
"client_addr": "10.50.62.163:0/478395830",
"tid": 195247
},
"events": [
{
"time": "2019-11-12 08:29:56.741549",
"event": "initiated"
},
{
"time": "2019-11-12 08:29:56.741549",
"event": "header_read"
},
{
"time": "2019-11-12 08:29:56.740440",
"event": "throttled"
},
{
"time": "2019-11-12 08:29:56.741553",
"event": "all_read"
},
{
"time": "2019-11-12 08:29:56.741555",
"event": "dispatched"
},
{
"time": "2019-11-12 08:29:56.741559",
"event": "queued_for_pg"
},
{
"time": "2019-11-12 08:29:56.741578",
"event": "reached_pg"
},
{
"time": "2019-11-12 08:29:56.741735",
"event": "started"
},
{
"time": "2019-11-12 08:29:56.742213",
"event": "sub_op_started"
},
{
"time": "2019-11-12 08:29:56.790062",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:08.484174",
"event": "commit_sent"
},
{
"time": "2019-11-12 08:30:08.484187",
"event": "done"
}
]
}
},
{
"description": "osd_op(client.71146410.0:90943 1.7s0
1:e0001313:::%2fc20%2fvx0000039%2fpot%2fsridhar_131%2fhls%2fh_74ee3ac80a0176f427a8ee7f2a9f922b%2fvar1058000%2fseg20271014_w1573280004.ts:head
[delete] snapc 0=[] ondisk+write+peerstat_old+known_if_redirected e30337)",
"initiated_at": "2019-11-12 08:29:58.109953",
"age": 77554.013277881997,
"duration": 10.374296185,
"type_data": {
"flag_point": "commit sent; apply or cleanup",
"client_info": {
"client": "client.71146410",
"client_addr": "10.50.62.164:0/3526276795",
"tid": 90943
},
"events": [
{
"time": "2019-11-12 08:29:58.109953",
"event": "initiated"
},
{
"time": "2019-11-12 08:29:58.109953",
"event": "header_read"
},
{
"time": "2019-11-12 08:29:58.109950",
"event": "throttled"
},
{
"time": "2019-11-12 08:29:58.109957",
"event": "all_read"
},
{
"time": "2019-11-12 08:29:58.109958",
"event": "dispatched"
},
{
"time": "2019-11-12 08:29:58.109962",
"event": "queued_for_pg"
},
{
"time": "2019-11-12 08:29:58.109978",
"event": "reached_pg"
},
{
"time": "2019-11-12 08:29:58.110122",
"event": "started"
},
{
"time": "2019-11-12 08:29:58.110241",
"event": "sub_op_started"
},
{
"time": "2019-11-12 08:29:58.128669",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:08.484235",
"event": "commit_sent"
},
{
"time": "2019-11-12 08:30:08.484249",
"event": "done"
}
]
}
}
]
}
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
