> On 23 Mar 2015, at 03:58, Haomai Wang <haomaiw...@gmail.com> wrote:
>
> On Mon, Mar 23, 2015 at 2:53 AM, Josef Johansson <jose...@gmail.com
> <mailto:jose...@gmail.com>> wrote:
>> Hi all!
>>
>> Trying to figure out how much my journals are used, using SSDs as journals
>> and SATA-drives as storage, I dive into perf dump.
>> But I can’t figure out why journal_queue_bytes is at constant 0. The only
>> thing that differs is dirtied in WBThrottle.
>
> journal_queue_bytes means how much journal data in the queue and is
> waiting for Journal Thread to be processed.
>
> Still now osd can't tell you how much data in the journal waiting for
> writeback and sync.
>
Hm, who knows that then?
Is this the WBThrottle value?
No way of knowing how much journal is used at all?
Maybe I thought of this wrong so if I understand you correctly
Data is written to OSD
The journal saves it to the queue
Waits for others to sync the requests as well
Sends a ACK to the client
Starts writing to the filestore buffer
filestore buffer commits when limits and met (inodes/ios-dirtied,
filestore_sync_max_interval)
So if I’m meeting latency and want to see if my journals are lazy, I should
indeed look at journal_queue_bytes, if that’s zero, it’s behaving well.
Thanks,
Josef
>>
>> Maybe I’ve disable that when setting the in-memory debug variables to 0/0?
>>
>> Thanks,
>> Josef
>>
>> # ceph --version
>> ceph version 0.67.7 (d7ab4244396b57aac8b7e80812115bbd079e6b73)
>>
>> # ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config show | grep
>> journal
>> "journaler": "0\/0",
>> "journal": "0\/0",
>> "journaler_allow_split_entries": "true",
>> "journaler_write_head_interval": "15",
>> "journaler_prefetch_periods": "10",
>> "journaler_prezero_periods": "5",
>> "journaler_batch_interval": "0.001",
>> "journaler_batch_max": "0",
>> "mds_kill_journal_at": "0",
>> "mds_kill_journal_expire_at": "0",
>> "mds_kill_journal_replay_at": "0",
>> "osd_journal": "\/var\/lib\/ceph\/osd\/ceph-0\/journal",
>> "osd_journal_size": "25600",
>> "filestore_fsync_flushes_journal_data": "false",
>> "filestore_journal_parallel": "false",
>> "filestore_journal_writeahead": "false",
>> "filestore_journal_trailing": "false",
>> "journal_dio": "true",
>> "journal_aio": "true",
>> "journal_force_aio": "false",
>> "journal_max_corrupt_search": "10485760",
>> "journal_block_align": "true",
>> "journal_write_header_frequency": "0",
>> "journal_max_write_bytes": "10485760",
>> "journal_max_write_entries": "100",
>> "journal_queue_max_ops": "300",
>> "journal_queue_max_bytes": "33554432",
>> "journal_align_min_size": "65536",
>> "journal_replay_from": "0",
>> "journal_zero_on_create": "false",
>> "journal_ignore_corruption": "false",
>>
>> # ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok perf dump
>> { "WBThrottle": { "bytes_dirtied": 32137216,
>> "bytes_wb": 0,
>> "ios_dirtied": 1445,
>> "ios_wb": 0,
>> "inodes_dirtied": 491,
>> "inodes_wb": 0},
>> "filestore": { "journal_queue_max_ops": 300,
>> "journal_queue_ops": 0,
>> "journal_ops": 116105073,
>> "journal_queue_max_bytes": 33554432,
>> "journal_queue_bytes": 0,
>> "journal_bytes": 3160504432839,
>> "journal_latency": { "avgcount": 116105073,
>> "sum": 64951.260611000},
>> "journal_wr": 112261141,
>> "journal_wr_bytes": { "avgcount": 112261141,
>> "sum": 3426141528064},
>> "op_queue_max_ops": 50,
>> "op_queue_ops": 0,
>> "ops": 116105073,
>> "op_queue_max_bytes": 104857600,
>> "op_queue_bytes": 0,
>> "bytes": 3159111228243,
>> "apply_latency": { "avgcount": 116105073,
>> "sum": 247410.066048000},
>> "committing": 0,
>> "commitcycle": 267176,
>> "commitcycle_interval": { "avgcount": 267176,
>> "sum": 1873193.631124000},
>> "commitcycle_latency": { "avgcount": 267176,
>> "sum": 390421.062990000},
>> "journal_full": 0,
>> "queue_transaction_latency_avg": { "avgcount": 116105073,
>> "sum": 378.948923000}},
>> "leveldb": { "leveldb_get": 699871216,
>> "leveldb_transaction": 522440246,
>> "leveldb_compact": 0,
>> "leveldb_compact_range": 0,
>> "leveldb_compact_queue_merge": 0,
>> "leveldb_compact_queue_len": 0},
>> "mutex-FileJournal::completions_lock": { "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "mutex-FileJournal::finisher_lock": { "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "mutex-FileJournal::write_lock": { "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "mutex-FileJournal::writeq_lock": { "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "mutex-JOS::ApplyManager::apply_lock": { "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "mutex-JOS::ApplyManager::com_lock": { "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "mutex-JOS::SubmitManager::lock": { "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "mutex-WBThrottle::lock": { "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "osd": { "opq": 0,
>> "op_wip": 0,
>> "op": 83920139,
>> "op_in_bytes": 1075345387581,
>> "op_out_bytes": 954428806331,
>> "op_latency": { "avgcount": 83920139,
>> "sum": 1279934.620502000},
>> "op_r": 32399024,
>> "op_r_out_bytes": 953657617715,
>> "op_r_latency": { "avgcount": 32399024,
>> "sum": 238792.729743000},
>> "op_w": 3321731,
>> "op_w_in_bytes": 52637941027,
>> "op_w_rlat": { "avgcount": 3321731,
>> "sum": 15577.620040000},
>> "op_w_latency": { "avgcount": 3321731,
>> "sum": 62541.746123000},
>> "op_rw": 48199384,
>> "op_rw_in_bytes": 1022707446554,
>> "op_rw_out_bytes": 771188616,
>> "op_rw_rlat": { "avgcount": 48199384,
>> "sum": 169776.087496000},
>> "op_rw_latency": { "avgcount": 48199384,
>> "sum": 978600.144636000},
>> "subop": 73746080,
>> "subop_in_bytes": 2008774955062,
>> "subop_latency": { "avgcount": 73746080,
>> "sum": 346096.627047000},
>> "subop_w": 0,
>> "subop_w_in_bytes": 2008774955062,
>> "subop_w_latency": { "avgcount": 73746080,
>> "sum": 346096.627047000},
>> "subop_pull": 0,
>> "subop_pull_latency": { "avgcount": 0,
>> "sum": 0.000000000},
>> "subop_push": 0,
>> "subop_push_in_bytes": 0,
>> "subop_push_latency": { "avgcount": 0,
>> "sum": 0.000000000},
>> "pull": 0,
>> "push": 8255,
>> "push_out_bytes": 24792364384,
>> "push_in": 0,
>> "push_in_bytes": 0,
>> "recovery_ops": 8126,
>> "loadavg": 585,
>> "buffer_bytes": 0,
>> "numpg": 402,
>> "numpg_primary": 161,
>> "numpg_replica": 241,
>> "numpg_stray": 0,
>> "heartbeat_to_peers": 38,
>> "heartbeat_from_peers": 0,
>> "map_messages": 15262,
>> "map_message_epochs": 70160,
>> "map_message_epoch_dups": 116821,
>> "messages_delayed_for_map": 105,
>> "peering_latency": { "avgcount": 628,
>> "sum": 28166.821628000}},
>> "throttle-filestore_bytes": { "val": 0,
>> "max": 33554432,
>> "get": 0,
>> "get_sum": 0,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 116105073,
>> "take_sum": 3160504432839,
>> "put": 112261141,
>> "put_sum": 3160504432839,
>> "wait": { "avgcount": 12,
>> "sum": 0.097590000}},
>> "throttle-filestore_ops": { "val": 0,
>> "max": 300,
>> "get": 0,
>> "get_sum": 0,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 116105073,
>> "take_sum": 116105073,
>> "put": 112261141,
>> "put_sum": 116105073,
>> "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "throttle-msgr_dispatch_throttler-client": { "val": 0,
>> "max": 104857600,
>> "get": 75187594,
>> "get_sum": 978560902803,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 0,
>> "take_sum": 0,
>> "put": 75187594,
>> "put_sum": 978560902803,
>> "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "throttle-msgr_dispatch_throttler-cluster": { "val": 0,
>> "max": 104857600,
>> "get": 159487799,
>> "get_sum": 2119870334346,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 0,
>> "take_sum": 0,
>> "put": 159487799,
>> "put_sum": 2119870334346,
>> "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0,
>> "max": 104857600,
>> "get": 19958614,
>> "get_sum": 938054858,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 0,
>> "take_sum": 0,
>> "put": 19958614,
>> "put_sum": 938054858,
>> "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0,
>> "max": 104857600,
>> "get": 19958614,
>> "get_sum": 938054858,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 0,
>> "take_sum": 0,
>> "put": 19958614,
>> "put_sum": 938054858,
>> "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "throttle-msgr_dispatch_throttler-hbclient": { "val": 0,
>> "max": 104857600,
>> "get": 44608572,
>> "get_sum": 2096602884,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 0,
>> "take_sum": 0,
>> "put": 44608572,
>> "put_sum": 2096602884,
>> "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "throttle-osd_client_bytes": { "val": 0,
>> "max": 524288000,
>> "get": 74786210,
>> "get_sum": 978238232914,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 0,
>> "take_sum": 0,
>> "put": 117138891,
>> "put_sum": 978238232914,
>> "wait": { "avgcount": 0,
>> "sum": 0.000000000}},
>> "throttle-osd_client_messages": { "val": 17,
>> "max": 100,
>> "get": 75909689,
>> "get_sum": 75909689,
>> "get_or_fail_fail": 0,
>> "get_or_fail_success": 0,
>> "take": 0,
>> "take_sum": 0,
>> "put": 75909672,
>> "put_sum": 75909672,
>> "wait": { "avgcount": 478562,
>> "sum": 5665.119305000}}}
>> _______________________________________________
>> ceph-users mailing list
>> ceph-users@lists.ceph.com
>> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>
>
>
> --
> Best Regards,
>
> Wheat
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
