Here you have. Perf dump:
{
"AsyncMessenger::Worker-0": {
"msgr_recv_messages": 12239872,
"msgr_send_messages": 12284221,
"msgr_recv_bytes": 43759275160,
"msgr_send_bytes": 61268769426,
"msgr_created_connections": 754,
"msgr_active_connections": 100,
"msgr_running_total_time": 939.476931816,
"msgr_running_send_time": 337.873686715,
"msgr_running_recv_time": 360.728238752,
"msgr_running_fast_dispatch_time": 183.737116872,
"msgr_send_messages_queue_lat": {
"avgcount": 12284206,
"sum": 1538.989479364,
"avgtime": 0.000125281
},
"msgr_handle_ack_lat": {
"avgcount": 5258403,
"sum": 1.005075918,
"avgtime": 0.000000191
}
},
"AsyncMessenger::Worker-1": {
"msgr_recv_messages": 12099771,
"msgr_send_messages": 12138795,
"msgr_recv_bytes": 56967534605,
"msgr_send_bytes": 130548664272,
"msgr_created_connections": 647,
"msgr_active_connections": 91,
"msgr_running_total_time": 977.277996439,
"msgr_running_send_time": 362.155959231,
"msgr_running_recv_time": 365.376281473,
"msgr_running_fast_dispatch_time": 191.186643292,
"msgr_send_messages_queue_lat": {
"avgcount": 12138818,
"sum": 1557.187685700,
"avgtime": 0.000128281
},
"msgr_handle_ack_lat": {
"avgcount": 6155265,
"sum": 1.096270527,
"avgtime": 0.000000178
}
},
"AsyncMessenger::Worker-2": {
"msgr_recv_messages": 11858354,
"msgr_send_messages": 11960404,
"msgr_recv_bytes": 60727084610,
"msgr_send_bytes": 168534726650,
"msgr_created_connections": 1043,
"msgr_active_connections": 103,
"msgr_running_total_time": 937.324084772,
"msgr_running_send_time": 351.174710644,
"msgr_running_recv_time": 2744.276782474,
"msgr_running_fast_dispatch_time": 172.960322050,
"msgr_send_messages_queue_lat": {
"avgcount": 11960392,
"sum": 1763.762581924,
"avgtime": 0.000147466
},
"msgr_handle_ack_lat": {
"avgcount": 2651457,
"sum": 0.538495450,
"avgtime": 0.000000203
}
},
"bluefs": {
"db_total_bytes": 128005955584,
"db_used_bytes": 3271557120,
"wal_total_bytes": 0,
"wal_used_bytes": 0,
"slow_total_bytes": 10000810377216,
"slow_used_bytes": 0,
"num_files": 70,
"log_bytes": 13045760,
"log_compactions": 58,
"logged_bytes": 922333184,
"files_written_wal": 2,
"files_written_sst": 13,
"bytes_written_wal": 1988489216,
"bytes_written_sst": 268890112,
"bytes_written_slow": 0,
"max_bytes_wal": 0,
"max_bytes_db": 3271557120,
"max_bytes_slow": 0,
"read_random_count": 577484,
"read_random_bytes": 2879541532,
"read_random_disk_count": 284290,
"read_random_disk_bytes": 1540394118,
"read_random_buffer_count": 319088,
"read_random_buffer_bytes": 1339147414,
"read_count": 1086625,
"read_bytes": 15054317429,
"read_prefetch_count": 1069462,
"read_prefetch_bytes": 14506469332,
"read_zeros_candidate": 0,
"read_zeros_errors": 0
},
"bluestore": {
"kv_flush_lat": {
"avgcount": 225099,
"sum": 526.605165277,
"avgtime": 0.002339438
},
"kv_commit_lat": {
"avgcount": 225099,
"sum": 61.412175620,
"avgtime": 0.000272822
},
"kv_sync_lat": {
"avgcount": 225099,
"sum": 588.017340897,
"avgtime": 0.002612261
},
"kv_final_lat": {
"avgcount": 225096,
"sum": 6.516869320,
"avgtime": 0.000028951
},
"state_prepare_lat": {
"avgcount": 241063,
"sum": 173.705759592,
"avgtime": 0.000720582
},
"state_aio_wait_lat": {
"avgcount": 241063,
"sum": 1008.936150524,
"avgtime": 0.004185362
},
"state_io_done_lat": {
"avgcount": 241063,
"sum": 2.923457351,
"avgtime": 0.000012127
},
"state_kv_queued_lat": {
"avgcount": 241063,
"sum": 560.050193021,
"avgtime": 0.002323252
},
"state_kv_commiting_lat": {
"avgcount": 241063,
"sum": 68.355225981,
"avgtime": 0.000283557
},
"state_kv_done_lat": {
"avgcount": 241063,
"sum": 0.097836444,
"avgtime": 0.000000405
},
"state_deferred_queued_lat": {
"avgcount": 47230,
"sum": 70711.960011607,
"avgtime": 1.497183146
},
"state_deferred_aio_wait_lat": {
"avgcount": 47230,
"sum": 70.931594687,
"avgtime": 0.001501833
},
"state_deferred_cleanup_lat": {
"avgcount": 47229,
"sum": 333702.384485631,
"avgtime": 7.065624605
},
"state_finishing_lat": {
"avgcount": 241062,
"sum": 0.033475853,
"avgtime": 0.000000138
},
"state_done_lat": {
"avgcount": 241062,
"sum": 15258.375631838,
"avgtime": 0.063296478
},
"throttle_lat": {
"avgcount": 241063,
"sum": 67.464167288,
"avgtime": 0.000279861
},
"submit_lat": {
"avgcount": 241063,
"sum": 190.468701838,
"avgtime": 0.000790120
},
"commit_lat": {
"avgcount": 241063,
"sum": 1814.026687331,
"avgtime": 0.007525114
},
"read_lat": {
"avgcount": 1590088,
"sum": 8928.843945649,
"avgtime": 0.005615314
},
"read_onode_meta_lat": {
"avgcount": 3106127,
"sum": 18.873640476,
"avgtime": 0.000006076
},
"read_wait_aio_lat": {
"avgcount": 1516039,
"sum": 8830.114048334,
"avgtime": 0.005824463
},
"compress_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"decompress_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"csum_lat": {
"avgcount": 7047778,
"sum": 52.140862362,
"avgtime": 0.000007398
},
"compress_success_count": 0,
"compress_rejected_count": 0,
"write_pad_bytes": 194870362,
"deferred_write_ops": 50669,
"deferred_write_bytes": 388481024,
"write_penalty_read_ops": 2067,
"bluestore_allocated": 294756155392,
"bluestore_stored": 292495538258,
"bluestore_compressed": 0,
"bluestore_compressed_allocated": 0,
"bluestore_compressed_original": 0,
"bluestore_onodes": 266002,
"bluestore_pinned_onodes": 1,
"onode_hits": 8782379,
"onode_misses": 6047682,
"onode_shard_hits": 1277688,
"bluestore_onode_shard_misses": 947296,
"bluestore_extents": 840549,
"bluestore_blobs": 840549,
"bluestore_buffers": 3591,
"bluestore_buffer_bytes": 224182272,
"bluestore_buffer_hit_bytes": 110213431882,
"bluestore_buffer_miss_bytes": 406742926152,
"bluestore_write_big": 39721,
"bluestore_write_big_bytes": 94625427456,
"bluestore_write_big_blobs": 1450312,
"bluestore_write_big_deferred": 0,
"bluestore_write_small": 60521,
"bluestore_write_small_bytes": 48842873,
"bluestore_write_small_unused": 926,
"bluestore_write_deferred": 62388,
"bluestore_write_deferred_bytes": 398618624,
"bluestore_write_small_pre_read": 7434,
"bluestore_write_new": 1448445,
"bluestore_txc": 241063,
"bluestore_onode_reshard": 27825,
"bluestore_blob_split": 5,
"bluestore_extent_compress": 3758,
"bluestore_gc_merged": 0,
"bluestore_read_eio": 0,
"bluestore_reads_with_retries": 0,
"bluestore_fragmentation_micros": 0,
"omap_seek_to_first_lat": {
"avgcount": 759176,
"sum": 8.351645295,
"avgtime": 0.000011000
},
"omap_upper_bound_lat": {
"avgcount": 308,
"sum": 0.002216099,
"avgtime": 0.000007195
},
"omap_lower_bound_lat": {
"avgcount": 87154,
"sum": 1.192486653,
"avgtime": 0.000013682
},
"omap_next_lat": {
"avgcount": 308598,
"sum": 0.211502859,
"avgtime": 0.000000685
},
"omap_get_keys_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"omap_get_values_lat": {
"avgcount": 19348,
"sum": 0.032918565,
"avgtime": 0.000001701
},
"clist_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"remove_lat": {
"avgcount": 239512,
"sum": 121.690074965,
"avgtime": 0.000508075
}
},
"bluestore-pricache": {
"target_bytes": 6713193267,
"mapped_bytes": 6718742528,
"unmapped_bytes": 467025920,
"heap_bytes": 7185768448,
"cache_bytes": 4161537138
},
"bluestore-pricache:data": {
"pri0_bytes": 0,
"pri1_bytes": 224182272,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 0,
"pri5_bytes": 0,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 30742099,
"reserved_bytes": 80619949,
"committed_bytes": 335544320
},
"bluestore-pricache:kv": {
"pri0_bytes": 0,
"pri1_bytes": 176540224,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 0,
"pri5_bytes": 0,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 230565745,
"reserved_bytes": 79433295,
"committed_bytes": 486539264
},
"bluestore-pricache:kv_onode": {
"pri0_bytes": 0,
"pri1_bytes": 1977148816,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 0,
"pri5_bytes": 0,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 20494732,
"reserved_bytes": 82731236,
"committed_bytes": 2080374784
},
"bluestore-pricache:meta": {
"pri0_bytes": 0,
"pri1_bytes": 942217320,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 0,
"pri5_bytes": 0,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 230565745,
"reserved_bytes": 68730919,
"committed_bytes": 1241513984
},
"cct": {
"total_workers": 5,
"unhealthy_workers": 0
},
"finisher-commit_finisher": {
"queue_len": 0,
"complete_latency": {
"avgcount": 1507,
"sum": 0.568110372,
"avgtime": 0.000376981
}
},
"finisher-objecter-finisher-0": {
"queue_len": 0,
"complete_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"mempool": {
"bloom_filter_bytes": 0,
"bloom_filter_items": 0,
"bluestore_alloc_bytes": 12461120,
"bluestore_alloc_items": 155764,
"bluestore_cache_data_bytes": 233998707,
"bluestore_cache_data_items": 229226,
"bluestore_cache_onode_bytes": 163857232,
"bluestore_cache_onode_items": 266002,
"bluestore_cache_meta_bytes": 456521298,
"bluestore_cache_meta_items": 84112717,
"bluestore_cache_other_bytes": 91560856,
"bluestore_cache_other_items": 11308804,
"bluestore_Buffer_bytes": 513888,
"bluestore_Buffer_items": 5353,
"bluestore_Extent_bytes": 40346352,
"bluestore_Extent_items": 840549,
"bluestore_Blob_bytes": 94141488,
"bluestore_Blob_items": 840549,
"bluestore_SharedBlob_bytes": 94141488,
"bluestore_SharedBlob_items": 840549,
"bluestore_inline_bl_bytes": 1134718,
"bluestore_inline_bl_items": 8758,
"bluestore_fsck_bytes": 0,
"bluestore_fsck_items": 0,
"bluestore_txc_bytes": 784,
"bluestore_txc_items": 1,
"bluestore_writing_deferred_bytes": 1918388,
"bluestore_writing_deferred_items": 81,
"bluestore_writing_bytes": 0,
"bluestore_writing_items": 0,
"bluefs_bytes": 36800,
"bluefs_items": 1443,
"bluefs_file_reader_bytes": 1842688,
"bluefs_file_reader_items": 95,
"bluefs_file_writer_bytes": 576,
"bluefs_file_writer_items": 3,
"buffer_anon_bytes": 10746933,
"buffer_anon_items": 25967,
"buffer_meta_bytes": 2541352,
"buffer_meta_items": 28879,
"osd_bytes": 916272,
"osd_items": 81,
"osd_mapbl_bytes": 0,
"osd_mapbl_items": 0,
"osd_pglog_bytes": 164733192,
"osd_pglog_items": 363565,
"osdmap_bytes": 2182392,
"osdmap_items": 53839,
"osdmap_mapping_bytes": 0,
"osdmap_mapping_items": 0,
"pgmap_bytes": 0,
"pgmap_items": 0,
"mds_co_bytes": 0,
"mds_co_items": 0,
"unittest_1_bytes": 0,
"unittest_1_items": 0,
"unittest_2_bytes": 0,
"unittest_2_items": 0
},
"objecter": {
"op_active": 0,
"op_laggy": 0,
"op_send": 0,
"op_send_bytes": 0,
"op_resend": 0,
"op_reply": 0,
"oplen_avg": {
"avgcount": 0,
"sum": 0
},
"op": 0,
"op_r": 0,
"op_w": 0,
"op_rmw": 0,
"op_pg": 0,
"osdop_stat": 0,
"osdop_create": 0,
"osdop_read": 0,
"osdop_write": 0,
"osdop_writefull": 0,
"osdop_writesame": 0,
"osdop_append": 0,
"osdop_zero": 0,
"osdop_truncate": 0,
"osdop_delete": 0,
"osdop_mapext": 0,
"osdop_sparse_read": 0,
"osdop_clonerange": 0,
"osdop_getxattr": 0,
"osdop_setxattr": 0,
"osdop_cmpxattr": 0,
"osdop_rmxattr": 0,
"osdop_resetxattrs": 0,
"osdop_call": 0,
"osdop_watch": 0,
"osdop_notify": 0,
"osdop_src_cmpxattr": 0,
"osdop_pgls": 0,
"osdop_pgls_filter": 0,
"osdop_other": 0,
"linger_active": 0,
"linger_send": 0,
"linger_resend": 0,
"linger_ping": 0,
"poolop_active": 0,
"poolop_send": 0,
"poolop_resend": 0,
"poolstat_active": 0,
"poolstat_send": 0,
"poolstat_resend": 0,
"statfs_active": 0,
"statfs_send": 0,
"statfs_resend": 0,
"command_active": 0,
"command_send": 0,
"command_resend": 0,
"map_epoch": 63038,
"map_full": 2,
"map_inc": 1511,
"osd_sessions": 0,
"osd_session_open": 0,
"osd_session_close": 0,
"osd_laggy": 0,
"omap_wr": 0,
"omap_rd": 0,
"omap_del": 0
},
"osd": {
"op_wip": 0,
"op": 272274,
"op_in_bytes": 39148504708,
"op_out_bytes": 181035613199,
"op_latency": {
"avgcount": 272274,
"sum": 2312.558437554,
"avgtime": 0.008493497
},
"op_process_latency": {
"avgcount": 272274,
"sum": 2151.138778914,
"avgtime": 0.007900639
},
"op_prepare_latency": {
"avgcount": 280169,
"sum": 1068.077633101,
"avgtime": 0.003812262
},
"op_r": 191667,
"op_r_out_bytes": 181035613199,
"op_r_latency": {
"avgcount": 191667,
"sum": 1035.691613777,
"avgtime": 0.005403599
},
"op_r_process_latency": {
"avgcount": 191667,
"sum": 959.303890513,
"avgtime": 0.005005055
},
"op_r_prepare_latency": {
"avgcount": 191712,
"sum": 962.370649266,
"avgtime": 0.005019876
},
"op_w": 34260,
"op_w_in_bytes": 39121651122,
"op_w_latency": {
"avgcount": 34260,
"sum": 1117.333684176,
"avgtime": 0.032613359
},
"op_w_process_latency": {
"avgcount": 34260,
"sum": 1059.202697781,
"avgtime": 0.030916599
},
"op_w_prepare_latency": {
"avgcount": 36798,
"sum": 91.420473715,
"avgtime": 0.002484387
},
"op_rw": 46347,
"op_rw_in_bytes": 26853586,
"op_rw_out_bytes": 0,
"op_rw_latency": {
"avgcount": 46347,
"sum": 159.533139601,
"avgtime": 0.003442145
},
"op_rw_process_latency": {
"avgcount": 46347,
"sum": 132.632190620,
"avgtime": 0.002861721
},
"op_rw_prepare_latency": {
"avgcount": 51659,
"sum": 14.286510120,
"avgtime": 0.000276554
},
"op_before_queue_op_lat": {
"avgcount": 897996,
"sum": 154.309110719,
"avgtime": 0.000171837
},
"op_before_dequeue_op_lat": {
"avgcount": 901696,
"sum": 622.443106332,
"avgtime": 0.000690302
},
"subop": 119722,
"subop_in_bytes": 55578084121,
"subop_latency": {
"avgcount": 119722,
"sum": 1159.173264760,
"avgtime": 0.009682207
},
"subop_w": 119722,
"subop_w_in_bytes": 55578084121,
"subop_w_latency": {
"avgcount": 119722,
"sum": 1159.173264760,
"avgtime": 0.009682207
},
"subop_pull": 0,
"subop_pull_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"subop_push": 0,
"subop_push_in_bytes": 0,
"subop_push_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"pull": 0,
"push": 86911,
"push_out_bytes": 34318722925,
"recovery_ops": 86922,
"recovery_bytes": 34314397051,
"loadavg": 62,
"cached_crc": 0,
"cached_crc_adjusted": 0,
"missed_crc": 0,
"numpg": 81,
"numpg_primary": 32,
"numpg_replica": 49,
"numpg_stray": 0,
"numpg_removing": 0,
"heartbeat_to_peers": 53,
"map_messages": 3924,
"map_message_epochs": 4465,
"map_message_epoch_dups": 2957,
"messages_delayed_for_map": 0,
"osd_map_cache_hit": 127507,
"osd_map_cache_miss": 22,
"osd_map_cache_miss_low": 2,
"osd_map_cache_miss_low_avg": {
"avgcount": 2,
"sum": 21
},
"osd_map_bl_cache_hit": 4254,
"osd_map_bl_cache_miss": 2798,
"stat_bytes": 10128816332800,
"stat_bytes_used": 422762119168,
"stat_bytes_avail": 9706054213632,
"copyfrom": 0,
"tier_promote": 0,
"tier_flush": 0,
"tier_flush_fail": 0,
"tier_try_flush": 0,
"tier_try_flush_fail": 0,
"tier_evict": 0,
"tier_whiteout": 0,
"tier_dirty": 29567,
"tier_clean": 0,
"tier_delay": 0,
"tier_proxy_read": 0,
"tier_proxy_write": 0,
"agent_wake": 0,
"agent_skip": 0,
"agent_flush": 0,
"agent_evict": 0,
"object_ctx_cache_hit": 288613,
"object_ctx_cache_total": 488855,
"op_cache_hit": 0,
"osd_tier_flush_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_tier_promote_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_tier_r_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_pg_info": 238380,
"osd_pg_fastinfo": 207321,
"osd_pg_biginfo": 1634
},
"recoverystate_perf": {
"initial_latency": {
"avgcount": 83,
"sum": 1.762260823,
"avgtime": 0.021232058
},
"started_latency": {
"avgcount": 597,
"sum": 12091840.383300855,
"avgtime": 20254.339000503
},
"reset_latency": {
"avgcount": 678,
"sum": 105.680233720,
"avgtime": 0.155870551
},
"start_latency": {
"avgcount": 678,
"sum": 0.013440001,
"avgtime": 0.000019823
},
"primary_latency": {
"avgcount": 229,
"sum": 3916876.117159230,
"avgtime": 17104.262520345
},
"peering_latency": {
"avgcount": 261,
"sum": 242.487618037,
"avgtime": 0.929071333
},
"backfilling_latency": {
"avgcount": 11,
"sum": 28355.181184224,
"avgtime": 2577.743744020
},
"waitremotebackfillreserved_latency": {
"avgcount": 11,
"sum": 0.010010103,
"avgtime": 0.000910009
},
"waitlocalbackfillreserved_latency": {
"avgcount": 11,
"sum": 0.000914005,
"avgtime": 0.000083091
},
"notbackfilling_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"repnotrecovering_latency": {
"avgcount": 236,
"sum": 8174666.478144644,
"avgtime": 34638.417280273
},
"repwaitrecoveryreserved_latency": {
"avgcount": 13,
"sum": 0.000544600,
"avgtime": 0.000041892
},
"repwaitbackfillreserved_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"reprecovering_latency": {
"avgcount": 13,
"sum": 5.236229915,
"avgtime": 0.402786916
},
"activating_latency": {
"avgcount": 226,
"sum": 7.792077960,
"avgtime": 0.034478221
},
"waitlocalrecoveryreserved_latency": {
"avgcount": 6,
"sum": 0.000532092,
"avgtime": 0.000088682
},
"waitremoterecoveryreserved_latency": {
"avgcount": 6,
"sum": 0.005132390,
"avgtime": 0.000855398
},
"recovering_latency": {
"avgcount": 6,
"sum": 2.188644385,
"avgtime": 0.364774064
},
"recovered_latency": {
"avgcount": 225,
"sum": 0.004724875,
"avgtime": 0.000020999
},
"clean_latency": {
"avgcount": 193,
"sum": 3888298.606566839,
"avgtime": 20146.624904491
},
"active_latency": {
"avgcount": 194,
"sum": 3916662.571342664,
"avgtime": 20188.982326508
},
"replicaactive_latency": {
"avgcount": 223,
"sum": 8174668.635187419,
"avgtime": 36657.706884248
},
"stray_latency": {
"avgcount": 417,
"sum": 343.521093120,
"avgtime": 0.823791590
},
"getinfo_latency": {
"avgcount": 261,
"sum": 26.466818426,
"avgtime": 0.101405434
},
"getlog_latency": {
"avgcount": 229,
"sum": 0.313088774,
"avgtime": 0.001367199
},
"waitactingchange_latency": {
"avgcount": 3,
"sum": 2.787477889,
"avgtime": 0.929159296
},
"incomplete_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"down_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"getmissing_latency": {
"avgcount": 226,
"sum": 0.070469958,
"avgtime": 0.000311813
},
"waitupthru_latency": {
"avgcount": 226,
"sum": 215.630016592,
"avgtime": 0.954115117
},
"notrecovering_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"rocksdb": {
"get": 3900573,
"get_latency": {
"avgcount": 3900573,
"sum": 46.497747616,
"avgtime": 0.000011920
},
"submit_latency": {
"avgcount": 241063,
"sum": 14.648569398,
"avgtime": 0.000060766
},
"submit_sync_latency": {
"avgcount": 225099,
"sum": 45.107860324,
"avgtime": 0.000200391
},
"compact": 0,
"compact_range": 0,
"compact_queue_merge": 0,
"compact_queue_len": 0,
"rocksdb_write_wal_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_memtable_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_delay_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_pre_and_post_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-bluestore_throttle_bytes": {
"val": 0,
"max": 67108864,
"get_started": 241063,
"get": 241063,
"get_sum": 1227078527883,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 225099,
"put_sum": 1227078527883,
"wait": {
"avgcount": 2039,
"sum": 65.096143721,
"avgtime": 0.031925524
}
},
"throttle-bluestore_throttle_deferred_bytes": {
"val": 0,
"max": 201326592,
"get_started": 0,
"get": 47230,
"get_sum": 32111982926,
"get_or_fail_fail": 0,
"get_or_fail_success": 47230,
"take": 0,
"take_sum": 0,
"put": 44540,
"put_sum": 32111982926,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-client": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 292633,
"get_sum": 39290356304,
"get_or_fail_fail": 0,
"get_or_fail_success": 292633,
"take": 0,
"take_sum": 0,
"put": 292633,
"put_sum": 39290356304,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-cluster": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 7182670,
"get_sum": 60512426404,
"get_or_fail_fail": 0,
"get_or_fail_success": 7182670,
"take": 0,
"take_sum": 0,
"put": 7182670,
"put_sum": 60512426404,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-hb_back_client": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 7382217,
"get_sum": 15008047161,
"get_or_fail_fail": 0,
"get_or_fail_success": 7382217,
"take": 0,
"take_sum": 0,
"put": 7382217,
"put_sum": 15008047161,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-hb_back_server": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 6979126,
"get_sum": 14188562814,
"get_or_fail_fail": 0,
"get_or_fail_success": 6979126,
"take": 0,
"take_sum": 0,
"put": 6979126,
"put_sum": 14188562814,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-hb_front_client": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 7382217,
"get_sum": 15008047161,
"get_or_fail_fail": 0,
"get_or_fail_success": 7382217,
"take": 0,
"take_sum": 0,
"put": 7382217,
"put_sum": 15008047161,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-hb_front_server": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 6979126,
"get_sum": 14188562814,
"get_or_fail_fail": 0,
"get_or_fail_success": 6979126,
"take": 0,
"take_sum": 0,
"put": 6979126,
"put_sum": 14188562814,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-ms_objecter": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-objecter_bytes": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-objecter_ops": {
"val": 0,
"max": 1024,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-osd_client_bytes": {
"val": 47,
"max": 524288000,
"get_started": 0,
"get": 291042,
"get_sum": 39289027913,
"get_or_fail_fail": 0,
"get_or_fail_success": 291042,
"take": 0,
"take_sum": 0,
"put": 1164143,
"put_sum": 39289027866,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-osd_client_messages": {
"val": 1,
"max": 256,
"get_started": 0,
"get": 291042,
"get_sum": 291042,
"get_or_fail_fail": 0,
"get_or_fail_success": 291042,
"take": 0,
"take_sum": 0,
"put": 291041,
"put_sum": 291041,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
}
}
and dump_mempools:
{
"mempool": {
"by_pool": {
"bloom_filter": {
"items": 0,
"bytes": 0
},
"bluestore_alloc": {
"items": 155779,
"bytes": 12462320
},
"bluestore_cache_data": {
"items": 228476,
"bytes": 233092536
},
"bluestore_cache_onode": {
"items": 265318,
"bytes": 163435888
},
"bluestore_cache_meta": {
"items": 83890049,
"bytes": 455300708
},
"bluestore_cache_other": {
"items": 11355469,
"bytes": 91930988
},
"bluestore_Buffer": {
"items": 5325,
"bytes": 511200
},
"bluestore_Extent": {
"items": 842524,
"bytes": 40441152
},
"bluestore_Blob": {
"items": 842524,
"bytes": 94362688
},
"bluestore_SharedBlob": {
"items": 842524,
"bytes": 94362688
},
"bluestore_inline_bl": {
"items": 8842,
"bytes": 1142714
},
"bluestore_fsck": {
"items": 0,
"bytes": 0
},
"bluestore_txc": {
"items": 0,
"bytes": 0
},
"bluestore_writing_deferred": {
"items": 77,
"bytes": 1798719
},
"bluestore_writing": {
"items": 0,
"bytes": 0
},
"bluefs": {
"items": 1443,
"bytes": 36800
},
"bluefs_file_reader": {
"items": 96,
"bytes": 2104832
},
"bluefs_file_writer": {
"items": 3,
"bytes": 576
},
"buffer_anon": {
"items": 25898,
"bytes": 10464534
},
"buffer_meta": {
"items": 28808,
"bytes": 2535104
},
"osd": {
"items": 81,
"bytes": 916272
},
"osd_mapbl": {
"items": 0,
"bytes": 0
},
"osd_pglog": {
"items": 363769,
"bytes": 164806408
},
"osdmap": {
"items": 53839,
"bytes": 2182392
},
"osdmap_mapping": {
"items": 0,
"bytes": 0
},
"pgmap": {
"items": 0,
"bytes": 0
},
"mds_co": {
"items": 0,
"bytes": 0
},
"unittest_1": {
"items": 0,
"bytes": 0
},
"unittest_2": {
"items": 0,
"bytes": 0
}
},
"total": {
"items": 98910844,
"bytes": 1371888519
}
}
}
Luis Domingues
Proton AG
------- Original Message -------
On Tuesday, July 11th, 2023 at 14:59, Mark Nelson <[email protected]> wrote:
> Hi Luis,
>
>
> Can you do a "ceph tell osd.<num> perf dump" and "ceph daemon osd.<num>
>
> dump_mempools"? Those should help us understand how much memory is
> being used by different parts of the OSD/bluestore and how much memory
> the priority cache thinks it has to work with.
>
>
> Mark
>
> On 7/11/23 4:57 AM, Luis Domingues wrote:
>
> > Hi everyone,
> >
> > We recently migrate a cluster from ceph-ansible to cephadm. Everything went
> > as expected.
> > But now we have some alerts on high memory usage. Cluster is running ceph
> > 16.2.13.
> >
> > Of course, after adoption OSDs ended up in the <unmanaged> zone:
> >
> > NAME PORTS RUNNING REFRESHED AGE PLACEMENT
> > osd 88 7m ago - <unmanaged>
> >
> > But the weirdest thing I observed, is that the OSDs seem to use more memory
> > that the mem limit:
> >
> > NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID
> > CONTAINER ID
> > osd.0 <node> running (5d) 2m ago 5d 19.7G 6400M 16.2.13 327f301eff51
> > ca07fe74a0fa
> > osd.1 <node> running (5d) 2m ago 5d 7068M 6400M 16.2.13 327f301eff51
> > 6223ed8e34e9
> > osd.10 <node> running (5d) 10m ago 5d 7235M 6400M 16.2.13 327f301eff51
> > 073ddc0d7391 osd.100 <node> running (5d) 2m ago 5d 7118M 6400M 16.2.13
> > 327f301eff51 b7f9238c0c24
> >
> > Does anybody knows why OSDs would use more memory than the limit?
> >
> > Thanks
> >
> > Luis Domingues
> > Proton AG
> > _______________________________________________
> > ceph-users mailing list -- [email protected]
> > To unsubscribe send an email to [email protected]
>
>
> --
> Best Regards,
> Mark Nelson
> Head of R&D (USA)
>
> Clyso GmbH
> p: +49 89 21552391 12
> a: Loristraße 8 | 80335 München | Germany
> w: https://clyso.com | e: [email protected]
>
> We are hiring: https://www.clyso.com/jobs/
> _______________________________________________
> ceph-users mailing list -- [email protected]
> To unsubscribe send an email to [email protected]
_______________________________________________
ceph-users mailing list -- [email protected]
To unsubscribe send an email to [email protected]
