[
https://issues.apache.org/jira/browse/SOLR-14669?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17169988#comment-17169988
]
Jörn Franke edited comment on SOLR-14669 at 8/3/20, 12:35 PM:
--------------------------------------------------------------
sorry for the late response. I confirm it is ZK 3.6.1 and Solr 8.6. It is a ZK
ensemble with 3 ZK nodes where problem occurs. Problem does not occur in
standalone mode.
leader
{code:java}
zk_version 3.6.1--104dcb3e3fb464b30c5186d229e00af9f332524b, built on
04/21/2020 15:01 GMT
zk_server_state leader
zk_peer_state leading - broadcast
zk_ephemerals_count 35
zk_num_alive_connections 7
zk_avg_latency 0.5851
zk_min_proposal_size 36
zk_outstanding_requests 0
zk_znode_count 790
zk_global_sessions 9
zk_last_client_response_size 158
zk_packets_sent 1250378
zk_packets_received 1250220
zk_max_proposal_size 1455
zk_max_client_response_size 36345565
zk_synced_followers 2
zk_connection_drop_probability 0.0
zk_watch_count 65
zk_synced_non_voting_followers 0
zk_synced_observers 0
zk_min_latency 0
zk_max_file_descriptor_count 65000
zk_approximate_data_size 488437179
zk_open_file_descriptor_count 74
zk_last_proposal_size 116
zk_pending_syncs 0
zk_leader_uptime 958675220
zk_local_sessions 0
zk_uptime 958676361
zk_max_latency 4744
zk_outstanding_tls_handshake 0
zk_learners 2
zk_min_client_response_size 16
zk_quorum_size 3
zk_proposal_count 304
zk_outstanding_changes_removed 411
zk_stale_requests_dropped 0
zk_large_requests_rejected 0
zk_connection_rejected 0
zk_sessionless_connections_expired 1
zk_looking_count 1
zk_dead_watchers_queued 0
zk_stale_requests 0
zk_connection_drop_count 1
zk_learner_proposal_received_count 0
zk_digest_mismatches_count 0
zk_dead_watchers_cleared 0
zk_response_packet_cache_hits 873109
zk_bytes_received_count 56311917
zk_add_dead_watcher_stall_time 0
zk_request_throttle_wait_count 0
zk_response_packet_cache_misses 1126
zk_ensemble_auth_success 0
zk_prep_processor_request_queued 1257581
zk_learner_commit_received_count 0
zk_stale_replies 0
zk_connection_request_count 12
zk_ensemble_auth_fail 0
zk_diff_count 2
zk_response_packet_get_children_cache_misses 0
zk_connection_revalidate_count 6
zk_quit_leading_due_to_disloyal_voter 0
zk_snap_count 1
zk_unrecoverable_error_count 0
zk_commit_count 304
zk_stale_sessions_expired 5
zk_response_packet_get_children_cache_hits 0
zk_sync_processor_request_queued 304
zk_outstanding_changes_queued 411
zk_request_commit_queued 304
zk_ensemble_auth_skip 0
zk_tls_handshake_exceeded 0
zk_revalidate_count 7
zk_avg_node_created_watch_count 0.0
zk_min_node_created_watch_count 0
zk_max_node_created_watch_count 0
zk_cnt_node_created_watch_count 0
zk_sum_node_created_watch_count 0
zk_avg_session_queues_drained 1.0
zk_min_session_queues_drained 1
zk_max_session_queues_drained 1
zk_cnt_session_queues_drained 304
zk_sum_session_queues_drained 304
zk_avg_write_commit_proc_req_queued 7.0E-4
zk_min_write_commit_proc_req_queued 0
zk_max_write_commit_proc_req_queued 5
zk_cnt_write_commit_proc_req_queued 1242288
zk_sum_write_commit_proc_req_queued 829
zk_avg_connection_token_deficit 0.0
zk_min_connection_token_deficit 0
zk_max_connection_token_deficit 0
zk_cnt_connection_token_deficit 12
zk_sum_connection_token_deficit 0
zk_avg_read_commit_proc_req_queued 1.0064
zk_min_read_commit_proc_req_queued 0
zk_max_read_commit_proc_req_queued 8
zk_cnt_read_commit_proc_req_queued 1242288
zk_sum_read_commit_proc_req_queued 1250250
zk_avg_node_deleted_watch_count 1.0
zk_min_node_deleted_watch_count 1
zk_max_node_deleted_watch_count 1
zk_cnt_node_deleted_watch_count 6
zk_sum_node_deleted_watch_count 6
zk_avg_startup_txns_load_time 17.0
zk_min_startup_txns_load_time 17
zk_max_startup_txns_load_time 17
zk_cnt_startup_txns_load_time 1
zk_sum_startup_txns_load_time 17
zk_avg_sync_processor_queue_size 0.0197
zk_min_sync_processor_queue_size 0
zk_max_sync_processor_queue_size 1
zk_cnt_sync_processor_queue_size 305
zk_sum_sync_processor_queue_size 6
zk_avg_follower_sync_time 0.0
zk_min_follower_sync_time 0
zk_max_follower_sync_time 0
zk_cnt_follower_sync_time 0
zk_sum_follower_sync_time 0
zk_avg_prep_processor_queue_size 0.0206
zk_min_prep_processor_queue_size 0
zk_max_prep_processor_queue_size 8
zk_cnt_prep_processor_queue_size 1257582
zk_sum_prep_processor_queue_size 25860
zk_avg_fsynctime 1.3087
zk_min_fsynctime 0
zk_max_fsynctime 28
zk_cnt_fsynctime 298
zk_sum_fsynctime 390
zk_avg_reads_issued_from_session_queue 0.4474
zk_min_reads_issued_from_session_queue 0
zk_max_reads_issued_from_session_queue 5
zk_cnt_reads_issued_from_session_queue 304
zk_sum_reads_issued_from_session_queue 136
zk_avg_snapshottime 1076.0
zk_min_snapshottime 1076
zk_max_snapshottime 1076
zk_cnt_snapshottime 1
zk_sum_snapshottime 1076
zk_avg_startup_txns_loaded 17.0
zk_min_startup_txns_loaded 17
zk_max_startup_txns_loaded 17
zk_cnt_startup_txns_loaded 1
zk_sum_startup_txns_loaded 17
zk_avg_reads_after_write_in_session_queue 0.4474
zk_min_reads_after_write_in_session_queue 0
zk_max_reads_after_write_in_session_queue 5
zk_cnt_reads_after_write_in_session_queue 304
zk_sum_reads_after_write_in_session_queue 136
zk_avg_requests_in_session_queue 1.6156
zk_min_requests_in_session_queue 1
zk_max_requests_in_session_queue 6
zk_cnt_requests_in_session_queue 744
zk_sum_requests_in_session_queue 1202
zk_avg_write_commit_proc_issued 1.0
zk_min_write_commit_proc_issued 1
zk_max_write_commit_proc_issued 1
zk_cnt_write_commit_proc_issued 304
zk_sum_write_commit_proc_issued 304
zk_avg_prep_process_time 0.1407
zk_min_prep_process_time 0
zk_max_prep_process_time 4744
zk_cnt_prep_process_time 1257581
zk_sum_prep_process_time 176892
zk_avg_pending_session_queue_size 1.023
zk_min_pending_session_queue_size 1
zk_max_pending_session_queue_size 3
zk_cnt_pending_session_queue_size 304
zk_sum_pending_session_queue_size 311
zk_avg_time_waiting_empty_pool_in_commit_processor_read_ms 0.0395
zk_min_time_waiting_empty_pool_in_commit_processor_read_ms 0
zk_max_time_waiting_empty_pool_in_commit_processor_read_ms 4
zk_cnt_time_waiting_empty_pool_in_commit_processor_read_ms 304
zk_sum_time_waiting_empty_pool_in_commit_processor_read_ms 12
zk_avg_commit_process_time 0.0111
zk_min_commit_process_time 0
zk_max_commit_process_time 5
zk_cnt_commit_process_time 1242288
zk_sum_commit_process_time 13829
zk_avg_dbinittime 2115.0
zk_min_dbinittime 2115
zk_max_dbinittime 2115
zk_cnt_dbinittime 1
zk_sum_dbinittime 2115
zk_avg_netty_queued_buffer_capacity 0.0
zk_min_netty_queued_buffer_capacity 0
zk_max_netty_queued_buffer_capacity 0
zk_cnt_netty_queued_buffer_capacity 0
zk_sum_netty_queued_buffer_capacity 0
zk_avg_election_time 31390.0
zk_min_election_time 31390
zk_max_election_time 31390
zk_cnt_election_time 1
zk_sum_election_time 31390
zk_avg_commit_commit_proc_req_queued 3.0E-4
zk_min_commit_commit_proc_req_queued 0
zk_max_commit_commit_proc_req_queued 2
zk_cnt_commit_commit_proc_req_queued 1242288
zk_sum_commit_commit_proc_req_queued 312
zk_avg_sync_processor_batch_size 1.0201
zk_min_sync_processor_batch_size 1
zk_max_sync_processor_batch_size 3
zk_cnt_sync_processor_batch_size 298
zk_sum_sync_processor_batch_size 304
zk_avg_node_children_watch_count 1.1864
zk_min_node_children_watch_count 1
zk_max_node_children_watch_count 4
zk_cnt_node_children_watch_count 59
zk_sum_node_children_watch_count 70
zk_avg_write_batch_time_in_commit_processor 0.3882
zk_min_write_batch_time_in_commit_processor 0
zk_max_write_batch_time_in_commit_processor 3
zk_cnt_write_batch_time_in_commit_processor 304
zk_sum_write_batch_time_in_commit_processor 118
zk_avg_read_commit_proc_issued 1.0061
zk_min_read_commit_proc_issued 0
zk_max_read_commit_proc_issued 8
zk_cnt_read_commit_proc_issued 1242288
zk_sum_read_commit_proc_issued 1249823
zk_avg_concurrent_request_processing_in_commit_processor 2.0
zk_min_concurrent_request_processing_in_commit_processor 1
zk_max_concurrent_request_processing_in_commit_processor 4
zk_cnt_concurrent_request_processing_in_commit_processor 10
zk_sum_concurrent_request_processing_in_commit_processor 20
zk_avg_node_changed_watch_count 1.9302
zk_min_node_changed_watch_count 1
zk_max_node_changed_watch_count 2
zk_cnt_node_changed_watch_count 43
zk_sum_node_changed_watch_count 83
zk_avg_sync_process_time 0.1053
zk_min_sync_process_time 0
zk_max_sync_process_time 2
zk_cnt_sync_process_time 304
zk_sum_sync_process_time 32
zk_avg_startup_snap_load_time 2045.0
zk_min_startup_snap_load_time 2045
zk_max_startup_snap_load_time 2045
zk_cnt_startup_snap_load_time 1
zk_sum_startup_snap_load_time 2045
zk_avg_prep_processor_queue_time_ms 0.0317
zk_min_prep_processor_queue_time_ms 0
zk_max_prep_processor_queue_time_ms 763
zk_cnt_prep_processor_queue_time_ms 1257581
zk_sum_prep_processor_queue_time_ms 39864
zk_p50_prep_processor_queue_time_ms 0
zk_p95_prep_processor_queue_time_ms 0
zk_p99_prep_processor_queue_time_ms 1
zk_p999_prep_processor_queue_time_ms 1
zk_avg_close_session_prep_time 0.1538
zk_min_close_session_prep_time 0
zk_max_close_session_prep_time 1
zk_cnt_close_session_prep_time 13
zk_sum_close_session_prep_time 2
zk_p50_close_session_prep_time 0
zk_p95_close_session_prep_time 1
zk_p99_close_session_prep_time 1
zk_p999_close_session_prep_time 1
zk_avg_read_commitproc_time_ms 0.0627
zk_min_read_commitproc_time_ms 0
zk_max_read_commitproc_time_ms 45
zk_cnt_read_commitproc_time_ms 1249959
zk_sum_read_commitproc_time_ms 78388
zk_p50_read_commitproc_time_ms 0
zk_p95_read_commitproc_time_ms 1
zk_p99_read_commitproc_time_ms 1
zk_p999_read_commitproc_time_ms 1
zk_avg_updatelatency 5.2763
zk_min_updatelatency 1
zk_max_updatelatency 34
zk_cnt_updatelatency 228
zk_sum_updatelatency 1203
zk_p50_updatelatency 4
zk_p95_updatelatency 15
zk_p99_updatelatency 22
zk_p999_updatelatency 34
zk_avg_local_write_committed_time_ms 0.1941
zk_min_local_write_committed_time_ms 0
zk_max_local_write_committed_time_ms 5
zk_cnt_local_write_committed_time_ms 304
zk_sum_local_write_committed_time_ms 59
zk_p50_local_write_committed_time_ms 0
zk_p95_local_write_committed_time_ms 1
zk_p99_local_write_committed_time_ms 4
zk_p999_local_write_committed_time_ms 5
zk_avg_readlatency 0.5842
zk_min_readlatency 0
zk_max_readlatency 4744
zk_cnt_readlatency 1249959
zk_sum_readlatency 730248
zk_p50_readlatency 1
zk_p95_readlatency 1
zk_p99_readlatency 1
zk_p999_readlatency 2
zk_avg_quorum_ack_latency 3.9572
zk_min_quorum_ack_latency 1
zk_max_quorum_ack_latency 32
zk_cnt_quorum_ack_latency 304
zk_sum_quorum_ack_latency 1203
zk_p50_quorum_ack_latency 3
zk_p95_quorum_ack_latency 12
zk_p99_quorum_ack_latency 20
zk_p999_quorum_ack_latency 32
zk_avg_om_commit_process_time_ms 0.0
zk_min_om_commit_process_time_ms 0
zk_max_om_commit_process_time_ms 0
zk_cnt_om_commit_process_time_ms 0
zk_sum_om_commit_process_time_ms 0
zk_p50_om_commit_process_time_ms 0
zk_p95_om_commit_process_time_ms 0
zk_p99_om_commit_process_time_ms 0
zk_p999_om_commit_process_time_ms 0
zk_avg_read_final_proc_time_ms 0.271
zk_min_read_final_proc_time_ms 0
zk_max_read_final_proc_time_ms 167
zk_cnt_read_final_proc_time_ms 1249959
zk_sum_read_final_proc_time_ms 338749
zk_p50_read_final_proc_time_ms 0
zk_p95_read_final_proc_time_ms 1
zk_p99_read_final_proc_time_ms 1
zk_p999_read_final_proc_time_ms 1
zk_avg_commit_propagation_latency 0.0
zk_min_commit_propagation_latency 0
zk_max_commit_propagation_latency 0
zk_cnt_commit_propagation_latency 0
zk_sum_commit_propagation_latency 0
zk_p50_commit_propagation_latency 0
zk_p95_commit_propagation_latency 0
zk_p99_commit_propagation_latency 0
zk_p999_commit_propagation_latency 0
zk_avg_dead_watchers_cleaner_latency 0.0
zk_min_dead_watchers_cleaner_latency 0
zk_max_dead_watchers_cleaner_latency 0
zk_cnt_dead_watchers_cleaner_latency 0
zk_sum_dead_watchers_cleaner_latency 0
zk_p50_dead_watchers_cleaner_latency 0
zk_p95_dead_watchers_cleaner_latency 0
zk_p99_dead_watchers_cleaner_latency 0
zk_p999_dead_watchers_cleaner_latency 0
zk_avg_write_final_proc_time_ms 0.3783
zk_min_write_final_proc_time_ms 0
zk_max_write_final_proc_time_ms 3
zk_cnt_write_final_proc_time_ms 304
zk_sum_write_final_proc_time_ms 115
zk_p50_write_final_proc_time_ms 0
zk_p95_write_final_proc_time_ms 1
zk_p99_write_final_proc_time_ms 2
zk_p999_write_final_proc_time_ms 3
zk_avg_proposal_ack_creation_latency 3.1546
zk_min_proposal_ack_creation_latency 1
zk_max_proposal_ack_creation_latency 32
zk_cnt_proposal_ack_creation_latency 304
zk_sum_proposal_ack_creation_latency 959
zk_p50_proposal_ack_creation_latency 2
zk_p95_proposal_ack_creation_latency 11
zk_p99_proposal_ack_creation_latency 19
zk_p999_proposal_ack_creation_latency 32
zk_avg_proposal_latency 0.0
zk_min_proposal_latency 0
zk_max_proposal_latency 0
zk_cnt_proposal_latency 0
zk_sum_proposal_latency 0
zk_p50_proposal_latency 0
zk_p95_proposal_latency 0
zk_p99_proposal_latency 0
zk_p999_proposal_latency 0
zk_avg_om_proposal_process_time_ms 0.0
zk_min_om_proposal_process_time_ms 0
zk_max_om_proposal_process_time_ms 0
zk_cnt_om_proposal_process_time_ms 0
zk_sum_om_proposal_process_time_ms 0
zk_p50_om_proposal_process_time_ms 0
zk_p95_om_proposal_process_time_ms 0
zk_p99_om_proposal_process_time_ms 0
zk_p999_om_proposal_process_time_ms 0
zk_avg_sync_processor_queue_and_flush_time_ms 2.375
zk_min_sync_processor_queue_and_flush_time_ms 0
zk_max_sync_processor_queue_and_flush_time_ms 30
zk_cnt_sync_processor_queue_and_flush_time_ms 304
zk_sum_sync_processor_queue_and_flush_time_ms 722
zk_p50_sync_processor_queue_and_flush_time_ms 1
zk_p95_sync_processor_queue_and_flush_time_ms 9
zk_p99_sync_processor_queue_and_flush_time_ms 18
zk_p999_sync_processor_queue_and_flush_time_ms 30
zk_avg_propagation_latency 4.553
zk_min_propagation_latency 1
zk_max_propagation_latency 34
zk_cnt_propagation_latency 302
zk_sum_propagation_latency 1375
zk_p50_propagation_latency 3
zk_p95_propagation_latency 13
zk_p99_propagation_latency 21
zk_p999_propagation_latency 34
zk_avg_server_write_committed_time_ms 0.0
zk_min_server_write_committed_time_ms 0
zk_max_server_write_committed_time_ms 0
zk_cnt_server_write_committed_time_ms 0
zk_sum_server_write_committed_time_ms 0
zk_p50_server_write_committed_time_ms 0
zk_p95_server_write_committed_time_ms 0
zk_p99_server_write_committed_time_ms 0
zk_p999_server_write_committed_time_ms 0
zk_avg_sync_processor_queue_time_ms 0.2467
zk_min_sync_processor_queue_time_ms 0
zk_max_sync_processor_queue_time_ms 9
zk_cnt_sync_processor_queue_time_ms 304
zk_sum_sync_processor_queue_time_ms 75
zk_p50_sync_processor_queue_time_ms 0
zk_p95_sync_processor_queue_time_ms 1
zk_p99_sync_processor_queue_time_ms 5
zk_p999_sync_processor_queue_time_ms 9
zk_avg_sync_processor_queue_flush_time_ms 1.9362
zk_min_sync_processor_queue_flush_time_ms 0
zk_max_sync_processor_queue_flush_time_ms 29
zk_cnt_sync_processor_queue_flush_time_ms 298
zk_sum_sync_processor_queue_flush_time_ms 577
zk_p50_sync_processor_queue_flush_time_ms 1
zk_p95_sync_processor_queue_flush_time_ms 7
zk_p99_sync_processor_queue_flush_time_ms 15
zk_p999_sync_processor_queue_flush_time_ms 29
zk_avg_write_commitproc_time_ms 3.6151
zk_min_write_commitproc_time_ms 1
zk_max_write_commitproc_time_ms 30
zk_cnt_write_commitproc_time_ms 304
zk_sum_write_commitproc_time_ms 1099
zk_p50_write_commitproc_time_ms 2
zk_p95_write_commitproc_time_ms 12
zk_p99_write_commitproc_time_ms 18
zk_p999_write_commitproc_time_ms 30
zk_avg_2_learner_handler_qp_size 0.0011
zk_min_2_learner_handler_qp_size 0
zk_max_2_learner_handler_qp_size 4
zk_cnt_2_learner_handler_qp_size 967377
zk_sum_2_learner_handler_qp_size 1050
zk_avg_3_learner_handler_qp_size 0.001
zk_min_3_learner_handler_qp_size 0
zk_max_3_learner_handler_qp_size 1
zk_cnt_3_learner_handler_qp_size 959565
zk_sum_3_learner_handler_qp_size 957
zk_avg_trufflesolr_read_per_namespace 733.0827
zk_min_trufflesolr_read_per_namespace 80
zk_max_trufflesolr_read_per_namespace 36345620
zk_cnt_trufflesolr_read_per_namespace 1015670
zk_sum_trufflesolr_read_per_namespace 744570072
zk_avg_zookeeper_read_per_namespace 250.0
zk_min_zookeeper_read_per_namespace 250
zk_max_zookeeper_read_per_namespace 250
zk_cnt_zookeeper_read_per_namespace 2
zk_sum_zookeeper_read_per_namespace 500
zk_avg_trufflesolr_write_per_namespace 347.3285
zk_min_trufflesolr_write_per_namespace 34
zk_max_trufflesolr_write_per_namespace 1323
zk_cnt_trufflesolr_write_per_namespace 277
zk_sum_trufflesolr_write_per_namespace 96210
zk_avg_zookeeper_write_per_namespace 182.0
zk_min_zookeeper_write_per_namespace 182
zk_max_zookeeper_write_per_namespace 182
zk_cnt_zookeeper_write_per_namespace 1
zk_sum_zookeeper_write_per_namespace 182
zk_avg_2_learner_handler_qp_time_ms 0.0041
zk_min_2_learner_handler_qp_time_ms 0
zk_max_2_learner_handler_qp_time_ms 3
zk_cnt_2_learner_handler_qp_time_ms 967
zk_sum_2_learner_handler_qp_time_ms 4
zk_p50_2_learner_handler_qp_time_ms 0
zk_p95_2_learner_handler_qp_time_ms 0
zk_p99_2_learner_handler_qp_time_ms 0
zk_p999_2_learner_handler_qp_time_ms 3
zk_avg_3_learner_handler_qp_time_ms 0.0
zk_min_3_learner_handler_qp_time_ms 0
zk_max_3_learner_handler_qp_time_ms 0
zk_cnt_3_learner_handler_qp_time_ms 959
zk_sum_3_learner_handler_qp_time_ms 0
zk_p50_3_learner_handler_qp_time_ms 0
zk_p95_3_learner_handler_qp_time_ms 0
zk_p99_3_learner_handler_qp_time_ms 0
zk_p999_3_learner_handler_qp_time_ms 0
{code}
follower
{code:java}
zk_version 3.6.1--104dcb3e3fb464b30c5186d229e00af9f332524b, built on
04/21/2020 15:01 GMT
zk_server_state follower
zk_peer_state following - broadcast
zk_ephemerals_count 35
zk_min_latency 0
zk_avg_latency 0.5746
zk_num_alive_connections 3
zk_max_file_descriptor_count 65000
zk_outstanding_requests 0
zk_approximate_data_size 488437179
zk_znode_count 790
zk_open_file_descriptor_count 68
zk_global_sessions 9
zk_local_sessions 0
zk_uptime 404349965
zk_last_client_response_size 16
zk_max_latency 1624
zk_packets_sent 77715
zk_outstanding_tls_handshake 0
zk_packets_received 77718
zk_max_client_response_size 1632
zk_connection_drop_probability 0.0
zk_watch_count 47
zk_min_client_response_size 16
zk_quorum_size 3
zk_proposal_count 0
zk_outstanding_changes_removed 0
zk_stale_requests_dropped 0
zk_large_requests_rejected 0
zk_connection_rejected 0
zk_sessionless_connections_expired 2
zk_looking_count 2
zk_dead_watchers_queued 0
zk_stale_requests 0
zk_connection_drop_count 0
zk_learner_proposal_received_count 304
zk_digest_mismatches_count 0
zk_dead_watchers_cleared 0
zk_response_packet_cache_hits 18058
zk_bytes_received_count 3179440
zk_add_dead_watcher_stall_time 0
zk_request_throttle_wait_count 0
zk_response_packet_cache_misses 39
zk_ensemble_auth_success 0
zk_prep_processor_request_queued 0
zk_learner_commit_received_count 304
zk_stale_replies 0
zk_connection_request_count 14
zk_ensemble_auth_fail 0
zk_diff_count 0
zk_response_packet_get_children_cache_misses 0
zk_connection_revalidate_count 6
zk_quit_leading_due_to_disloyal_voter 0
zk_snap_count 0
zk_unrecoverable_error_count 0
zk_commit_count 0
zk_stale_sessions_expired 0
zk_response_packet_get_children_cache_hits 0
zk_sync_processor_request_queued 305
zk_outstanding_changes_queued 0
zk_request_commit_queued 7621
zk_ensemble_auth_skip 0
zk_tls_handshake_exceeded 0
zk_revalidate_count 0
zk_avg_node_created_watch_count 0.0
zk_min_node_created_watch_count 0
zk_max_node_created_watch_count 0
zk_cnt_node_created_watch_count 0
zk_sum_node_created_watch_count 0
zk_avg_session_queues_drained 0.9618
zk_min_session_queues_drained 0
zk_max_session_queues_drained 1
zk_cnt_session_queues_drained 7621
zk_sum_session_queues_drained 7330
zk_avg_write_commit_proc_req_queued 0.1217
zk_min_write_commit_proc_req_queued 0
zk_max_write_commit_proc_req_queued 3
zk_cnt_write_commit_proc_req_queued 181792
zk_sum_write_commit_proc_req_queued 22117
zk_avg_connection_token_deficit 0.0
zk_min_connection_token_deficit 0
zk_max_connection_token_deficit 0
zk_cnt_connection_token_deficit 14
zk_sum_connection_token_deficit 0
zk_avg_read_commit_proc_req_queued 0.9711
zk_min_read_commit_proc_req_queued 0
zk_max_read_commit_proc_req_queued 5
zk_cnt_read_commit_proc_req_queued 181792
zk_sum_read_commit_proc_req_queued 176544
zk_avg_node_deleted_watch_count 0.0
zk_min_node_deleted_watch_count 0
zk_max_node_deleted_watch_count 0
zk_cnt_node_deleted_watch_count 0
zk_sum_node_deleted_watch_count 0
zk_avg_startup_txns_load_time 15.0
zk_min_startup_txns_load_time 15
zk_max_startup_txns_load_time 15
zk_cnt_startup_txns_load_time 1
zk_sum_startup_txns_load_time 15
zk_avg_sync_processor_queue_size 0.0456
zk_min_sync_processor_queue_size 0
zk_max_sync_processor_queue_size 2
zk_cnt_sync_processor_queue_size 307
zk_sum_sync_processor_queue_size 14
zk_avg_follower_sync_time 56.0
zk_min_follower_sync_time 56
zk_max_follower_sync_time 56
zk_cnt_follower_sync_time 2
zk_sum_follower_sync_time 112
zk_avg_prep_processor_queue_size 0.0
zk_min_prep_processor_queue_size 0
zk_max_prep_processor_queue_size 0
zk_cnt_prep_processor_queue_size 0
zk_sum_prep_processor_queue_size 0
zk_avg_fsynctime 1.5918
zk_min_fsynctime 0
zk_max_fsynctime 106
zk_cnt_fsynctime 294
zk_sum_fsynctime 468
zk_avg_reads_issued_from_session_queue 0.9616
zk_min_reads_issued_from_session_queue 0
zk_max_reads_issued_from_session_queue 2
zk_cnt_reads_issued_from_session_queue 7621
zk_sum_reads_issued_from_session_queue 7328
zk_avg_snapshottime 0.0
zk_min_snapshottime 0
zk_max_snapshottime 0
zk_cnt_snapshottime 0
zk_sum_snapshottime 0
zk_avg_startup_txns_loaded 16.0
zk_min_startup_txns_loaded 16
zk_max_startup_txns_loaded 16
zk_cnt_startup_txns_loaded 1
zk_sum_startup_txns_loaded 16
zk_avg_reads_after_write_in_session_queue 0.9997
zk_min_reads_after_write_in_session_queue 0
zk_max_reads_after_write_in_session_queue 2
zk_cnt_reads_after_write_in_session_queue 7330
zk_sum_reads_after_write_in_session_queue 7328
zk_avg_requests_in_session_queue 1.7319
zk_min_requests_in_session_queue 1
zk_max_requests_in_session_queue 6
zk_cnt_requests_in_session_queue 21988
zk_sum_requests_in_session_queue 38082
zk_avg_write_commit_proc_issued 1.0
zk_min_write_commit_proc_issued 1
zk_max_write_commit_proc_issued 1
zk_cnt_write_commit_proc_issued 7621
zk_sum_write_commit_proc_issued 7621
zk_avg_prep_process_time 0.0
zk_min_prep_process_time 0
zk_max_prep_process_time 0
zk_cnt_prep_process_time 0
zk_sum_prep_process_time 0
zk_avg_pending_session_queue_size 1.0
zk_min_pending_session_queue_size 1
zk_max_pending_session_queue_size 1
zk_cnt_pending_session_queue_size 7330
zk_sum_pending_session_queue_size 7330
zk_avg_time_waiting_empty_pool_in_commit_processor_read_ms 0.0115
zk_min_time_waiting_empty_pool_in_commit_processor_read_ms 0
zk_max_time_waiting_empty_pool_in_commit_processor_read_ms 1
zk_cnt_time_waiting_empty_pool_in_commit_processor_read_ms 7621
zk_sum_time_waiting_empty_pool_in_commit_processor_read_ms 88
zk_avg_commit_process_time 0.0197
zk_min_commit_process_time 0
zk_max_commit_process_time 11
zk_cnt_commit_process_time 181792
zk_sum_commit_process_time 3586
zk_avg_dbinittime 5338.0
zk_min_dbinittime 5338
zk_max_dbinittime 5338
zk_cnt_dbinittime 1
zk_sum_dbinittime 5338
zk_avg_netty_queued_buffer_capacity 0.0
zk_min_netty_queued_buffer_capacity 0
zk_max_netty_queued_buffer_capacity 0
zk_cnt_netty_queued_buffer_capacity 0
zk_sum_netty_queued_buffer_capacity 0
zk_avg_election_time 151.0
zk_min_election_time 20
zk_max_election_time 282
zk_cnt_election_time 2
zk_sum_election_time 302
zk_avg_commit_commit_proc_req_queued 0.0422
zk_min_commit_commit_proc_req_queued 0
zk_max_commit_commit_proc_req_queued 3
zk_cnt_commit_commit_proc_req_queued 181792
zk_sum_commit_commit_proc_req_queued 7674
zk_avg_sync_processor_batch_size 1.0374
zk_min_sync_processor_batch_size 1
zk_max_sync_processor_batch_size 3
zk_cnt_sync_processor_batch_size 294
zk_sum_sync_processor_batch_size 305
zk_avg_node_children_watch_count 3.75
zk_min_node_children_watch_count 3
zk_max_node_children_watch_count 4
zk_cnt_node_children_watch_count 4
zk_sum_node_children_watch_count 15
zk_avg_write_batch_time_in_commit_processor 0.1476
zk_min_write_batch_time_in_commit_processor 0
zk_max_write_batch_time_in_commit_processor 11
zk_cnt_write_batch_time_in_commit_processor 7621
zk_sum_write_batch_time_in_commit_processor 1125
zk_avg_read_commit_proc_issued 0.8871
zk_min_read_commit_proc_issued 0
zk_max_read_commit_proc_issued 5
zk_cnt_read_commit_proc_issued 181792
zk_sum_read_commit_proc_issued 161263
zk_avg_concurrent_request_processing_in_commit_processor 1.0207
zk_min_concurrent_request_processing_in_commit_processor 1
zk_max_concurrent_request_processing_in_commit_processor 2
zk_cnt_concurrent_request_processing_in_commit_processor 242
zk_sum_concurrent_request_processing_in_commit_processor 247
zk_avg_node_changed_watch_count 0.0
zk_min_node_changed_watch_count 0
zk_max_node_changed_watch_count 0
zk_cnt_node_changed_watch_count 0
zk_sum_node_changed_watch_count 0
zk_avg_sync_process_time 0.0918
zk_min_sync_process_time 0
zk_max_sync_process_time 1
zk_cnt_sync_process_time 305
zk_sum_sync_process_time 28
zk_avg_startup_snap_load_time 5268.0
zk_min_startup_snap_load_time 5268
zk_max_startup_snap_load_time 5268
zk_cnt_startup_snap_load_time 1
zk_sum_startup_snap_load_time 5268
zk_avg_prep_processor_queue_time_ms 0.0
zk_min_prep_processor_queue_time_ms 0
zk_max_prep_processor_queue_time_ms 0
zk_cnt_prep_processor_queue_time_ms 0
zk_sum_prep_processor_queue_time_ms 0
zk_p50_prep_processor_queue_time_ms 0
zk_p95_prep_processor_queue_time_ms 0
zk_p99_prep_processor_queue_time_ms 0
zk_p999_prep_processor_queue_time_ms 0
zk_avg_close_session_prep_time 0.0
zk_min_close_session_prep_time 0
zk_max_close_session_prep_time 0
zk_cnt_close_session_prep_time 0
zk_sum_close_session_prep_time 0
zk_p50_close_session_prep_time 0
zk_p95_close_session_prep_time 0
zk_p99_close_session_prep_time 0
zk_p999_close_session_prep_time 0
zk_avg_read_commitproc_time_ms 0.8202
zk_min_read_commitproc_time_ms 0
zk_max_read_commitproc_time_ms 13334
zk_cnt_read_commitproc_time_ms 168591
zk_sum_read_commitproc_time_ms 138271
zk_p50_read_commitproc_time_ms 0
zk_p95_read_commitproc_time_ms 1
zk_p99_read_commitproc_time_ms 1
zk_p999_read_commitproc_time_ms 2
zk_avg_updatelatency 6.0714
zk_min_updatelatency 3
zk_max_updatelatency 13
zk_cnt_updatelatency 14
zk_sum_updatelatency 85
zk_p50_updatelatency 5
zk_p95_updatelatency 13
zk_p99_updatelatency 13
zk_p999_updatelatency 13
zk_avg_local_write_committed_time_ms 0.0482
zk_min_local_write_committed_time_ms 0
zk_max_local_write_committed_time_ms 5
zk_cnt_local_write_committed_time_ms 7330
zk_sum_local_write_committed_time_ms 353
zk_p50_local_write_committed_time_ms 0
zk_p95_local_write_committed_time_ms 0
zk_p99_local_write_committed_time_ms 1
zk_p999_local_write_committed_time_ms 1
zk_avg_readlatency 1.2633
zk_min_readlatency 0
zk_max_readlatency 13334
zk_cnt_readlatency 175907
zk_sum_readlatency 222226
zk_p50_readlatency 0
zk_p95_readlatency 1
zk_p99_readlatency 1
zk_p999_readlatency 15
zk_avg_quorum_ack_latency 0.0
zk_min_quorum_ack_latency 0
zk_max_quorum_ack_latency 0
zk_cnt_quorum_ack_latency 0
zk_sum_quorum_ack_latency 0
zk_p50_quorum_ack_latency 0
zk_p95_quorum_ack_latency 0
zk_p99_quorum_ack_latency 0
zk_p999_quorum_ack_latency 0
zk_avg_om_commit_process_time_ms 0.0
zk_min_om_commit_process_time_ms 0
zk_max_om_commit_process_time_ms 0
zk_cnt_om_commit_process_time_ms 0
zk_sum_om_commit_process_time_ms 0
zk_p50_om_commit_process_time_ms 0
zk_p95_om_commit_process_time_ms 0
zk_p99_om_commit_process_time_ms 0
zk_p999_om_commit_process_time_ms 0
zk_avg_read_final_proc_time_ms 0.1748
zk_min_read_final_proc_time_ms 0
zk_max_read_final_proc_time_ms 59
zk_cnt_read_final_proc_time_ms 168591
zk_sum_read_final_proc_time_ms 29471
zk_p50_read_final_proc_time_ms 0
zk_p95_read_final_proc_time_ms 1
zk_p99_read_final_proc_time_ms 1
zk_p999_read_final_proc_time_ms 1
zk_avg_commit_propagation_latency 342.8393
zk_min_commit_propagation_latency 1
zk_max_commit_propagation_latency 103233
zk_cnt_commit_propagation_latency 305
zk_sum_commit_propagation_latency 104566
zk_p50_commit_propagation_latency 3
zk_p95_commit_propagation_latency 14
zk_p99_commit_propagation_latency 21
zk_p999_commit_propagation_latency 103233
zk_avg_dead_watchers_cleaner_latency 0.0
zk_min_dead_watchers_cleaner_latency 0
zk_max_dead_watchers_cleaner_latency 0
zk_cnt_dead_watchers_cleaner_latency 0
zk_sum_dead_watchers_cleaner_latency 0
zk_p50_dead_watchers_cleaner_latency 0
zk_p95_dead_watchers_cleaner_latency 0
zk_p99_dead_watchers_cleaner_latency 0
zk_p999_dead_watchers_cleaner_latency 0
zk_avg_write_final_proc_time_ms 0.1401
zk_min_write_final_proc_time_ms 0
zk_max_write_final_proc_time_ms 11
zk_cnt_write_final_proc_time_ms 7621
zk_sum_write_final_proc_time_ms 1068
zk_p50_write_final_proc_time_ms 0
zk_p95_write_final_proc_time_ms 1
zk_p99_write_final_proc_time_ms 1
zk_p999_write_final_proc_time_ms 1
zk_avg_proposal_ack_creation_latency 342.5574
zk_min_proposal_ack_creation_latency 1
zk_max_proposal_ack_creation_latency 103236
zk_cnt_proposal_ack_creation_latency 305
zk_sum_proposal_ack_creation_latency 104480
zk_p50_proposal_ack_creation_latency 2
zk_p95_proposal_ack_creation_latency 12
zk_p99_proposal_ack_creation_latency 91
zk_p999_proposal_ack_creation_latency 103236
zk_avg_proposal_latency 1.3849
zk_min_proposal_latency 0
zk_max_proposal_latency 18
zk_cnt_proposal_latency 304
zk_sum_proposal_latency 421
zk_p50_proposal_latency 1
zk_p95_proposal_latency 3
zk_p99_proposal_latency 10
zk_p999_proposal_latency 18
zk_avg_om_proposal_process_time_ms 0.0
zk_min_om_proposal_process_time_ms 0
zk_max_om_proposal_process_time_ms 0
zk_cnt_om_proposal_process_time_ms 0
zk_sum_om_proposal_process_time_ms 0
zk_p50_om_proposal_process_time_ms 0
zk_p95_om_proposal_process_time_ms 0
zk_p99_om_proposal_process_time_ms 0
zk_p999_om_proposal_process_time_ms 0
zk_avg_sync_processor_queue_and_flush_time_ms 2.6852
zk_min_sync_processor_queue_and_flush_time_ms 0
zk_max_sync_processor_queue_and_flush_time_ms 106
zk_cnt_sync_processor_queue_and_flush_time_ms 305
zk_sum_sync_processor_queue_and_flush_time_ms 819
zk_p50_sync_processor_queue_and_flush_time_ms 1
zk_p95_sync_processor_queue_and_flush_time_ms 9
zk_p99_sync_processor_queue_and_flush_time_ms 28
zk_p999_sync_processor_queue_and_flush_time_ms 106
zk_avg_propagation_latency 344.8125
zk_min_propagation_latency 1
zk_max_propagation_latency 103235
zk_cnt_propagation_latency 304
zk_sum_propagation_latency 104823
zk_p50_propagation_latency 4
zk_p95_propagation_latency 15
zk_p99_propagation_latency 22
zk_p999_propagation_latency 103235
zk_avg_server_write_committed_time_ms 0.3058
zk_min_server_write_committed_time_ms 0
zk_max_server_write_committed_time_ms 10
zk_cnt_server_write_committed_time_ms 291
zk_sum_server_write_committed_time_ms 89
zk_p50_server_write_committed_time_ms 0
zk_p95_server_write_committed_time_ms 1
zk_p99_server_write_committed_time_ms 8
zk_p999_server_write_committed_time_ms 10
zk_avg_sync_processor_queue_time_ms 0.3443
zk_min_sync_processor_queue_time_ms 0
zk_max_sync_processor_queue_time_ms 11
zk_cnt_sync_processor_queue_time_ms 305
zk_sum_sync_processor_queue_time_ms 105
zk_p50_sync_processor_queue_time_ms 0
zk_p95_sync_processor_queue_time_ms 2
zk_p99_sync_processor_queue_time_ms 7
zk_p999_sync_processor_queue_time_ms 11
zk_avg_sync_processor_queue_flush_time_ms 2.2245
zk_min_sync_processor_queue_flush_time_ms 0
zk_max_sync_processor_queue_flush_time_ms 106
zk_cnt_sync_processor_queue_flush_time_ms 294
zk_sum_sync_processor_queue_flush_time_ms 654
zk_p50_sync_processor_queue_flush_time_ms 1
zk_p95_sync_processor_queue_flush_time_ms 4
zk_p99_sync_processor_queue_flush_time_ms 31
zk_p999_sync_processor_queue_flush_time_ms 106
zk_avg_write_commitproc_time_ms 0.5658
zk_min_write_commitproc_time_ms 0
zk_max_write_commitproc_time_ms 20
zk_cnt_write_commitproc_time_ms 7330
zk_sum_write_commitproc_time_ms 4147
zk_p50_write_commitproc_time_ms 1
zk_p95_write_commitproc_time_ms 1
zk_p99_write_commitproc_time_ms 2
zk_p999_write_commitproc_time_ms 4
zk_avg_trufflesolr_read_per_namespace 303.6124
zk_min_trufflesolr_read_per_namespace 80
zk_max_trufflesolr_read_per_namespace 3361
zk_cnt_trufflesolr_read_per_namespace 22339
zk_sum_trufflesolr_read_per_namespace 6782397
zk_avg_trufflesolr_write_per_namespace 347.3285
zk_min_trufflesolr_write_per_namespace 34
zk_max_trufflesolr_write_per_namespace 1323
zk_cnt_trufflesolr_write_per_namespace 277
zk_sum_trufflesolr_write_per_namespace 96210
zk_avg_zookeeper_write_per_namespace 182.0
zk_min_zookeeper_write_per_namespace 182
zk_max_zookeeper_write_per_namespace 182
zk_cnt_zookeeper_write_per_namespace 3
zk_sum_zookeeper_write_per_namespace 546
{code}
was (Author: jornfranke):
sorry for the late response. I confirm it is ZK 3.6.1 and Solr 8.6. It is a ZK
ensemble, problem does not occur in standalone mode.
leader
{code:java}
zk_version 3.6.1--104dcb3e3fb464b30c5186d229e00af9f332524b, built on
04/21/2020 15:01 GMT
zk_server_state leader
zk_peer_state leading - broadcast
zk_ephemerals_count 35
zk_num_alive_connections 7
zk_avg_latency 0.5851
zk_min_proposal_size 36
zk_outstanding_requests 0
zk_znode_count 790
zk_global_sessions 9
zk_last_client_response_size 158
zk_packets_sent 1250378
zk_packets_received 1250220
zk_max_proposal_size 1455
zk_max_client_response_size 36345565
zk_synced_followers 2
zk_connection_drop_probability 0.0
zk_watch_count 65
zk_synced_non_voting_followers 0
zk_synced_observers 0
zk_min_latency 0
zk_max_file_descriptor_count 65000
zk_approximate_data_size 488437179
zk_open_file_descriptor_count 74
zk_last_proposal_size 116
zk_pending_syncs 0
zk_leader_uptime 958675220
zk_local_sessions 0
zk_uptime 958676361
zk_max_latency 4744
zk_outstanding_tls_handshake 0
zk_learners 2
zk_min_client_response_size 16
zk_quorum_size 3
zk_proposal_count 304
zk_outstanding_changes_removed 411
zk_stale_requests_dropped 0
zk_large_requests_rejected 0
zk_connection_rejected 0
zk_sessionless_connections_expired 1
zk_looking_count 1
zk_dead_watchers_queued 0
zk_stale_requests 0
zk_connection_drop_count 1
zk_learner_proposal_received_count 0
zk_digest_mismatches_count 0
zk_dead_watchers_cleared 0
zk_response_packet_cache_hits 873109
zk_bytes_received_count 56311917
zk_add_dead_watcher_stall_time 0
zk_request_throttle_wait_count 0
zk_response_packet_cache_misses 1126
zk_ensemble_auth_success 0
zk_prep_processor_request_queued 1257581
zk_learner_commit_received_count 0
zk_stale_replies 0
zk_connection_request_count 12
zk_ensemble_auth_fail 0
zk_diff_count 2
zk_response_packet_get_children_cache_misses 0
zk_connection_revalidate_count 6
zk_quit_leading_due_to_disloyal_voter 0
zk_snap_count 1
zk_unrecoverable_error_count 0
zk_commit_count 304
zk_stale_sessions_expired 5
zk_response_packet_get_children_cache_hits 0
zk_sync_processor_request_queued 304
zk_outstanding_changes_queued 411
zk_request_commit_queued 304
zk_ensemble_auth_skip 0
zk_tls_handshake_exceeded 0
zk_revalidate_count 7
zk_avg_node_created_watch_count 0.0
zk_min_node_created_watch_count 0
zk_max_node_created_watch_count 0
zk_cnt_node_created_watch_count 0
zk_sum_node_created_watch_count 0
zk_avg_session_queues_drained 1.0
zk_min_session_queues_drained 1
zk_max_session_queues_drained 1
zk_cnt_session_queues_drained 304
zk_sum_session_queues_drained 304
zk_avg_write_commit_proc_req_queued 7.0E-4
zk_min_write_commit_proc_req_queued 0
zk_max_write_commit_proc_req_queued 5
zk_cnt_write_commit_proc_req_queued 1242288
zk_sum_write_commit_proc_req_queued 829
zk_avg_connection_token_deficit 0.0
zk_min_connection_token_deficit 0
zk_max_connection_token_deficit 0
zk_cnt_connection_token_deficit 12
zk_sum_connection_token_deficit 0
zk_avg_read_commit_proc_req_queued 1.0064
zk_min_read_commit_proc_req_queued 0
zk_max_read_commit_proc_req_queued 8
zk_cnt_read_commit_proc_req_queued 1242288
zk_sum_read_commit_proc_req_queued 1250250
zk_avg_node_deleted_watch_count 1.0
zk_min_node_deleted_watch_count 1
zk_max_node_deleted_watch_count 1
zk_cnt_node_deleted_watch_count 6
zk_sum_node_deleted_watch_count 6
zk_avg_startup_txns_load_time 17.0
zk_min_startup_txns_load_time 17
zk_max_startup_txns_load_time 17
zk_cnt_startup_txns_load_time 1
zk_sum_startup_txns_load_time 17
zk_avg_sync_processor_queue_size 0.0197
zk_min_sync_processor_queue_size 0
zk_max_sync_processor_queue_size 1
zk_cnt_sync_processor_queue_size 305
zk_sum_sync_processor_queue_size 6
zk_avg_follower_sync_time 0.0
zk_min_follower_sync_time 0
zk_max_follower_sync_time 0
zk_cnt_follower_sync_time 0
zk_sum_follower_sync_time 0
zk_avg_prep_processor_queue_size 0.0206
zk_min_prep_processor_queue_size 0
zk_max_prep_processor_queue_size 8
zk_cnt_prep_processor_queue_size 1257582
zk_sum_prep_processor_queue_size 25860
zk_avg_fsynctime 1.3087
zk_min_fsynctime 0
zk_max_fsynctime 28
zk_cnt_fsynctime 298
zk_sum_fsynctime 390
zk_avg_reads_issued_from_session_queue 0.4474
zk_min_reads_issued_from_session_queue 0
zk_max_reads_issued_from_session_queue 5
zk_cnt_reads_issued_from_session_queue 304
zk_sum_reads_issued_from_session_queue 136
zk_avg_snapshottime 1076.0
zk_min_snapshottime 1076
zk_max_snapshottime 1076
zk_cnt_snapshottime 1
zk_sum_snapshottime 1076
zk_avg_startup_txns_loaded 17.0
zk_min_startup_txns_loaded 17
zk_max_startup_txns_loaded 17
zk_cnt_startup_txns_loaded 1
zk_sum_startup_txns_loaded 17
zk_avg_reads_after_write_in_session_queue 0.4474
zk_min_reads_after_write_in_session_queue 0
zk_max_reads_after_write_in_session_queue 5
zk_cnt_reads_after_write_in_session_queue 304
zk_sum_reads_after_write_in_session_queue 136
zk_avg_requests_in_session_queue 1.6156
zk_min_requests_in_session_queue 1
zk_max_requests_in_session_queue 6
zk_cnt_requests_in_session_queue 744
zk_sum_requests_in_session_queue 1202
zk_avg_write_commit_proc_issued 1.0
zk_min_write_commit_proc_issued 1
zk_max_write_commit_proc_issued 1
zk_cnt_write_commit_proc_issued 304
zk_sum_write_commit_proc_issued 304
zk_avg_prep_process_time 0.1407
zk_min_prep_process_time 0
zk_max_prep_process_time 4744
zk_cnt_prep_process_time 1257581
zk_sum_prep_process_time 176892
zk_avg_pending_session_queue_size 1.023
zk_min_pending_session_queue_size 1
zk_max_pending_session_queue_size 3
zk_cnt_pending_session_queue_size 304
zk_sum_pending_session_queue_size 311
zk_avg_time_waiting_empty_pool_in_commit_processor_read_ms 0.0395
zk_min_time_waiting_empty_pool_in_commit_processor_read_ms 0
zk_max_time_waiting_empty_pool_in_commit_processor_read_ms 4
zk_cnt_time_waiting_empty_pool_in_commit_processor_read_ms 304
zk_sum_time_waiting_empty_pool_in_commit_processor_read_ms 12
zk_avg_commit_process_time 0.0111
zk_min_commit_process_time 0
zk_max_commit_process_time 5
zk_cnt_commit_process_time 1242288
zk_sum_commit_process_time 13829
zk_avg_dbinittime 2115.0
zk_min_dbinittime 2115
zk_max_dbinittime 2115
zk_cnt_dbinittime 1
zk_sum_dbinittime 2115
zk_avg_netty_queued_buffer_capacity 0.0
zk_min_netty_queued_buffer_capacity 0
zk_max_netty_queued_buffer_capacity 0
zk_cnt_netty_queued_buffer_capacity 0
zk_sum_netty_queued_buffer_capacity 0
zk_avg_election_time 31390.0
zk_min_election_time 31390
zk_max_election_time 31390
zk_cnt_election_time 1
zk_sum_election_time 31390
zk_avg_commit_commit_proc_req_queued 3.0E-4
zk_min_commit_commit_proc_req_queued 0
zk_max_commit_commit_proc_req_queued 2
zk_cnt_commit_commit_proc_req_queued 1242288
zk_sum_commit_commit_proc_req_queued 312
zk_avg_sync_processor_batch_size 1.0201
zk_min_sync_processor_batch_size 1
zk_max_sync_processor_batch_size 3
zk_cnt_sync_processor_batch_size 298
zk_sum_sync_processor_batch_size 304
zk_avg_node_children_watch_count 1.1864
zk_min_node_children_watch_count 1
zk_max_node_children_watch_count 4
zk_cnt_node_children_watch_count 59
zk_sum_node_children_watch_count 70
zk_avg_write_batch_time_in_commit_processor 0.3882
zk_min_write_batch_time_in_commit_processor 0
zk_max_write_batch_time_in_commit_processor 3
zk_cnt_write_batch_time_in_commit_processor 304
zk_sum_write_batch_time_in_commit_processor 118
zk_avg_read_commit_proc_issued 1.0061
zk_min_read_commit_proc_issued 0
zk_max_read_commit_proc_issued 8
zk_cnt_read_commit_proc_issued 1242288
zk_sum_read_commit_proc_issued 1249823
zk_avg_concurrent_request_processing_in_commit_processor 2.0
zk_min_concurrent_request_processing_in_commit_processor 1
zk_max_concurrent_request_processing_in_commit_processor 4
zk_cnt_concurrent_request_processing_in_commit_processor 10
zk_sum_concurrent_request_processing_in_commit_processor 20
zk_avg_node_changed_watch_count 1.9302
zk_min_node_changed_watch_count 1
zk_max_node_changed_watch_count 2
zk_cnt_node_changed_watch_count 43
zk_sum_node_changed_watch_count 83
zk_avg_sync_process_time 0.1053
zk_min_sync_process_time 0
zk_max_sync_process_time 2
zk_cnt_sync_process_time 304
zk_sum_sync_process_time 32
zk_avg_startup_snap_load_time 2045.0
zk_min_startup_snap_load_time 2045
zk_max_startup_snap_load_time 2045
zk_cnt_startup_snap_load_time 1
zk_sum_startup_snap_load_time 2045
zk_avg_prep_processor_queue_time_ms 0.0317
zk_min_prep_processor_queue_time_ms 0
zk_max_prep_processor_queue_time_ms 763
zk_cnt_prep_processor_queue_time_ms 1257581
zk_sum_prep_processor_queue_time_ms 39864
zk_p50_prep_processor_queue_time_ms 0
zk_p95_prep_processor_queue_time_ms 0
zk_p99_prep_processor_queue_time_ms 1
zk_p999_prep_processor_queue_time_ms 1
zk_avg_close_session_prep_time 0.1538
zk_min_close_session_prep_time 0
zk_max_close_session_prep_time 1
zk_cnt_close_session_prep_time 13
zk_sum_close_session_prep_time 2
zk_p50_close_session_prep_time 0
zk_p95_close_session_prep_time 1
zk_p99_close_session_prep_time 1
zk_p999_close_session_prep_time 1
zk_avg_read_commitproc_time_ms 0.0627
zk_min_read_commitproc_time_ms 0
zk_max_read_commitproc_time_ms 45
zk_cnt_read_commitproc_time_ms 1249959
zk_sum_read_commitproc_time_ms 78388
zk_p50_read_commitproc_time_ms 0
zk_p95_read_commitproc_time_ms 1
zk_p99_read_commitproc_time_ms 1
zk_p999_read_commitproc_time_ms 1
zk_avg_updatelatency 5.2763
zk_min_updatelatency 1
zk_max_updatelatency 34
zk_cnt_updatelatency 228
zk_sum_updatelatency 1203
zk_p50_updatelatency 4
zk_p95_updatelatency 15
zk_p99_updatelatency 22
zk_p999_updatelatency 34
zk_avg_local_write_committed_time_ms 0.1941
zk_min_local_write_committed_time_ms 0
zk_max_local_write_committed_time_ms 5
zk_cnt_local_write_committed_time_ms 304
zk_sum_local_write_committed_time_ms 59
zk_p50_local_write_committed_time_ms 0
zk_p95_local_write_committed_time_ms 1
zk_p99_local_write_committed_time_ms 4
zk_p999_local_write_committed_time_ms 5
zk_avg_readlatency 0.5842
zk_min_readlatency 0
zk_max_readlatency 4744
zk_cnt_readlatency 1249959
zk_sum_readlatency 730248
zk_p50_readlatency 1
zk_p95_readlatency 1
zk_p99_readlatency 1
zk_p999_readlatency 2
zk_avg_quorum_ack_latency 3.9572
zk_min_quorum_ack_latency 1
zk_max_quorum_ack_latency 32
zk_cnt_quorum_ack_latency 304
zk_sum_quorum_ack_latency 1203
zk_p50_quorum_ack_latency 3
zk_p95_quorum_ack_latency 12
zk_p99_quorum_ack_latency 20
zk_p999_quorum_ack_latency 32
zk_avg_om_commit_process_time_ms 0.0
zk_min_om_commit_process_time_ms 0
zk_max_om_commit_process_time_ms 0
zk_cnt_om_commit_process_time_ms 0
zk_sum_om_commit_process_time_ms 0
zk_p50_om_commit_process_time_ms 0
zk_p95_om_commit_process_time_ms 0
zk_p99_om_commit_process_time_ms 0
zk_p999_om_commit_process_time_ms 0
zk_avg_read_final_proc_time_ms 0.271
zk_min_read_final_proc_time_ms 0
zk_max_read_final_proc_time_ms 167
zk_cnt_read_final_proc_time_ms 1249959
zk_sum_read_final_proc_time_ms 338749
zk_p50_read_final_proc_time_ms 0
zk_p95_read_final_proc_time_ms 1
zk_p99_read_final_proc_time_ms 1
zk_p999_read_final_proc_time_ms 1
zk_avg_commit_propagation_latency 0.0
zk_min_commit_propagation_latency 0
zk_max_commit_propagation_latency 0
zk_cnt_commit_propagation_latency 0
zk_sum_commit_propagation_latency 0
zk_p50_commit_propagation_latency 0
zk_p95_commit_propagation_latency 0
zk_p99_commit_propagation_latency 0
zk_p999_commit_propagation_latency 0
zk_avg_dead_watchers_cleaner_latency 0.0
zk_min_dead_watchers_cleaner_latency 0
zk_max_dead_watchers_cleaner_latency 0
zk_cnt_dead_watchers_cleaner_latency 0
zk_sum_dead_watchers_cleaner_latency 0
zk_p50_dead_watchers_cleaner_latency 0
zk_p95_dead_watchers_cleaner_latency 0
zk_p99_dead_watchers_cleaner_latency 0
zk_p999_dead_watchers_cleaner_latency 0
zk_avg_write_final_proc_time_ms 0.3783
zk_min_write_final_proc_time_ms 0
zk_max_write_final_proc_time_ms 3
zk_cnt_write_final_proc_time_ms 304
zk_sum_write_final_proc_time_ms 115
zk_p50_write_final_proc_time_ms 0
zk_p95_write_final_proc_time_ms 1
zk_p99_write_final_proc_time_ms 2
zk_p999_write_final_proc_time_ms 3
zk_avg_proposal_ack_creation_latency 3.1546
zk_min_proposal_ack_creation_latency 1
zk_max_proposal_ack_creation_latency 32
zk_cnt_proposal_ack_creation_latency 304
zk_sum_proposal_ack_creation_latency 959
zk_p50_proposal_ack_creation_latency 2
zk_p95_proposal_ack_creation_latency 11
zk_p99_proposal_ack_creation_latency 19
zk_p999_proposal_ack_creation_latency 32
zk_avg_proposal_latency 0.0
zk_min_proposal_latency 0
zk_max_proposal_latency 0
zk_cnt_proposal_latency 0
zk_sum_proposal_latency 0
zk_p50_proposal_latency 0
zk_p95_proposal_latency 0
zk_p99_proposal_latency 0
zk_p999_proposal_latency 0
zk_avg_om_proposal_process_time_ms 0.0
zk_min_om_proposal_process_time_ms 0
zk_max_om_proposal_process_time_ms 0
zk_cnt_om_proposal_process_time_ms 0
zk_sum_om_proposal_process_time_ms 0
zk_p50_om_proposal_process_time_ms 0
zk_p95_om_proposal_process_time_ms 0
zk_p99_om_proposal_process_time_ms 0
zk_p999_om_proposal_process_time_ms 0
zk_avg_sync_processor_queue_and_flush_time_ms 2.375
zk_min_sync_processor_queue_and_flush_time_ms 0
zk_max_sync_processor_queue_and_flush_time_ms 30
zk_cnt_sync_processor_queue_and_flush_time_ms 304
zk_sum_sync_processor_queue_and_flush_time_ms 722
zk_p50_sync_processor_queue_and_flush_time_ms 1
zk_p95_sync_processor_queue_and_flush_time_ms 9
zk_p99_sync_processor_queue_and_flush_time_ms 18
zk_p999_sync_processor_queue_and_flush_time_ms 30
zk_avg_propagation_latency 4.553
zk_min_propagation_latency 1
zk_max_propagation_latency 34
zk_cnt_propagation_latency 302
zk_sum_propagation_latency 1375
zk_p50_propagation_latency 3
zk_p95_propagation_latency 13
zk_p99_propagation_latency 21
zk_p999_propagation_latency 34
zk_avg_server_write_committed_time_ms 0.0
zk_min_server_write_committed_time_ms 0
zk_max_server_write_committed_time_ms 0
zk_cnt_server_write_committed_time_ms 0
zk_sum_server_write_committed_time_ms 0
zk_p50_server_write_committed_time_ms 0
zk_p95_server_write_committed_time_ms 0
zk_p99_server_write_committed_time_ms 0
zk_p999_server_write_committed_time_ms 0
zk_avg_sync_processor_queue_time_ms 0.2467
zk_min_sync_processor_queue_time_ms 0
zk_max_sync_processor_queue_time_ms 9
zk_cnt_sync_processor_queue_time_ms 304
zk_sum_sync_processor_queue_time_ms 75
zk_p50_sync_processor_queue_time_ms 0
zk_p95_sync_processor_queue_time_ms 1
zk_p99_sync_processor_queue_time_ms 5
zk_p999_sync_processor_queue_time_ms 9
zk_avg_sync_processor_queue_flush_time_ms 1.9362
zk_min_sync_processor_queue_flush_time_ms 0
zk_max_sync_processor_queue_flush_time_ms 29
zk_cnt_sync_processor_queue_flush_time_ms 298
zk_sum_sync_processor_queue_flush_time_ms 577
zk_p50_sync_processor_queue_flush_time_ms 1
zk_p95_sync_processor_queue_flush_time_ms 7
zk_p99_sync_processor_queue_flush_time_ms 15
zk_p999_sync_processor_queue_flush_time_ms 29
zk_avg_write_commitproc_time_ms 3.6151
zk_min_write_commitproc_time_ms 1
zk_max_write_commitproc_time_ms 30
zk_cnt_write_commitproc_time_ms 304
zk_sum_write_commitproc_time_ms 1099
zk_p50_write_commitproc_time_ms 2
zk_p95_write_commitproc_time_ms 12
zk_p99_write_commitproc_time_ms 18
zk_p999_write_commitproc_time_ms 30
zk_avg_2_learner_handler_qp_size 0.0011
zk_min_2_learner_handler_qp_size 0
zk_max_2_learner_handler_qp_size 4
zk_cnt_2_learner_handler_qp_size 967377
zk_sum_2_learner_handler_qp_size 1050
zk_avg_3_learner_handler_qp_size 0.001
zk_min_3_learner_handler_qp_size 0
zk_max_3_learner_handler_qp_size 1
zk_cnt_3_learner_handler_qp_size 959565
zk_sum_3_learner_handler_qp_size 957
zk_avg_trufflesolr_read_per_namespace 733.0827
zk_min_trufflesolr_read_per_namespace 80
zk_max_trufflesolr_read_per_namespace 36345620
zk_cnt_trufflesolr_read_per_namespace 1015670
zk_sum_trufflesolr_read_per_namespace 744570072
zk_avg_zookeeper_read_per_namespace 250.0
zk_min_zookeeper_read_per_namespace 250
zk_max_zookeeper_read_per_namespace 250
zk_cnt_zookeeper_read_per_namespace 2
zk_sum_zookeeper_read_per_namespace 500
zk_avg_trufflesolr_write_per_namespace 347.3285
zk_min_trufflesolr_write_per_namespace 34
zk_max_trufflesolr_write_per_namespace 1323
zk_cnt_trufflesolr_write_per_namespace 277
zk_sum_trufflesolr_write_per_namespace 96210
zk_avg_zookeeper_write_per_namespace 182.0
zk_min_zookeeper_write_per_namespace 182
zk_max_zookeeper_write_per_namespace 182
zk_cnt_zookeeper_write_per_namespace 1
zk_sum_zookeeper_write_per_namespace 182
zk_avg_2_learner_handler_qp_time_ms 0.0041
zk_min_2_learner_handler_qp_time_ms 0
zk_max_2_learner_handler_qp_time_ms 3
zk_cnt_2_learner_handler_qp_time_ms 967
zk_sum_2_learner_handler_qp_time_ms 4
zk_p50_2_learner_handler_qp_time_ms 0
zk_p95_2_learner_handler_qp_time_ms 0
zk_p99_2_learner_handler_qp_time_ms 0
zk_p999_2_learner_handler_qp_time_ms 3
zk_avg_3_learner_handler_qp_time_ms 0.0
zk_min_3_learner_handler_qp_time_ms 0
zk_max_3_learner_handler_qp_time_ms 0
zk_cnt_3_learner_handler_qp_time_ms 959
zk_sum_3_learner_handler_qp_time_ms 0
zk_p50_3_learner_handler_qp_time_ms 0
zk_p95_3_learner_handler_qp_time_ms 0
zk_p99_3_learner_handler_qp_time_ms 0
zk_p999_3_learner_handler_qp_time_ms 0
{code}
follower
{code:java}
zk_version 3.6.1--104dcb3e3fb464b30c5186d229e00af9f332524b, built on
04/21/2020 15:01 GMT
zk_server_state follower
zk_peer_state following - broadcast
zk_ephemerals_count 35
zk_min_latency 0
zk_avg_latency 0.5746
zk_num_alive_connections 3
zk_max_file_descriptor_count 65000
zk_outstanding_requests 0
zk_approximate_data_size 488437179
zk_znode_count 790
zk_open_file_descriptor_count 68
zk_global_sessions 9
zk_local_sessions 0
zk_uptime 404349965
zk_last_client_response_size 16
zk_max_latency 1624
zk_packets_sent 77715
zk_outstanding_tls_handshake 0
zk_packets_received 77718
zk_max_client_response_size 1632
zk_connection_drop_probability 0.0
zk_watch_count 47
zk_min_client_response_size 16
zk_quorum_size 3
zk_proposal_count 0
zk_outstanding_changes_removed 0
zk_stale_requests_dropped 0
zk_large_requests_rejected 0
zk_connection_rejected 0
zk_sessionless_connections_expired 2
zk_looking_count 2
zk_dead_watchers_queued 0
zk_stale_requests 0
zk_connection_drop_count 0
zk_learner_proposal_received_count 304
zk_digest_mismatches_count 0
zk_dead_watchers_cleared 0
zk_response_packet_cache_hits 18058
zk_bytes_received_count 3179440
zk_add_dead_watcher_stall_time 0
zk_request_throttle_wait_count 0
zk_response_packet_cache_misses 39
zk_ensemble_auth_success 0
zk_prep_processor_request_queued 0
zk_learner_commit_received_count 304
zk_stale_replies 0
zk_connection_request_count 14
zk_ensemble_auth_fail 0
zk_diff_count 0
zk_response_packet_get_children_cache_misses 0
zk_connection_revalidate_count 6
zk_quit_leading_due_to_disloyal_voter 0
zk_snap_count 0
zk_unrecoverable_error_count 0
zk_commit_count 0
zk_stale_sessions_expired 0
zk_response_packet_get_children_cache_hits 0
zk_sync_processor_request_queued 305
zk_outstanding_changes_queued 0
zk_request_commit_queued 7621
zk_ensemble_auth_skip 0
zk_tls_handshake_exceeded 0
zk_revalidate_count 0
zk_avg_node_created_watch_count 0.0
zk_min_node_created_watch_count 0
zk_max_node_created_watch_count 0
zk_cnt_node_created_watch_count 0
zk_sum_node_created_watch_count 0
zk_avg_session_queues_drained 0.9618
zk_min_session_queues_drained 0
zk_max_session_queues_drained 1
zk_cnt_session_queues_drained 7621
zk_sum_session_queues_drained 7330
zk_avg_write_commit_proc_req_queued 0.1217
zk_min_write_commit_proc_req_queued 0
zk_max_write_commit_proc_req_queued 3
zk_cnt_write_commit_proc_req_queued 181792
zk_sum_write_commit_proc_req_queued 22117
zk_avg_connection_token_deficit 0.0
zk_min_connection_token_deficit 0
zk_max_connection_token_deficit 0
zk_cnt_connection_token_deficit 14
zk_sum_connection_token_deficit 0
zk_avg_read_commit_proc_req_queued 0.9711
zk_min_read_commit_proc_req_queued 0
zk_max_read_commit_proc_req_queued 5
zk_cnt_read_commit_proc_req_queued 181792
zk_sum_read_commit_proc_req_queued 176544
zk_avg_node_deleted_watch_count 0.0
zk_min_node_deleted_watch_count 0
zk_max_node_deleted_watch_count 0
zk_cnt_node_deleted_watch_count 0
zk_sum_node_deleted_watch_count 0
zk_avg_startup_txns_load_time 15.0
zk_min_startup_txns_load_time 15
zk_max_startup_txns_load_time 15
zk_cnt_startup_txns_load_time 1
zk_sum_startup_txns_load_time 15
zk_avg_sync_processor_queue_size 0.0456
zk_min_sync_processor_queue_size 0
zk_max_sync_processor_queue_size 2
zk_cnt_sync_processor_queue_size 307
zk_sum_sync_processor_queue_size 14
zk_avg_follower_sync_time 56.0
zk_min_follower_sync_time 56
zk_max_follower_sync_time 56
zk_cnt_follower_sync_time 2
zk_sum_follower_sync_time 112
zk_avg_prep_processor_queue_size 0.0
zk_min_prep_processor_queue_size 0
zk_max_prep_processor_queue_size 0
zk_cnt_prep_processor_queue_size 0
zk_sum_prep_processor_queue_size 0
zk_avg_fsynctime 1.5918
zk_min_fsynctime 0
zk_max_fsynctime 106
zk_cnt_fsynctime 294
zk_sum_fsynctime 468
zk_avg_reads_issued_from_session_queue 0.9616
zk_min_reads_issued_from_session_queue 0
zk_max_reads_issued_from_session_queue 2
zk_cnt_reads_issued_from_session_queue 7621
zk_sum_reads_issued_from_session_queue 7328
zk_avg_snapshottime 0.0
zk_min_snapshottime 0
zk_max_snapshottime 0
zk_cnt_snapshottime 0
zk_sum_snapshottime 0
zk_avg_startup_txns_loaded 16.0
zk_min_startup_txns_loaded 16
zk_max_startup_txns_loaded 16
zk_cnt_startup_txns_loaded 1
zk_sum_startup_txns_loaded 16
zk_avg_reads_after_write_in_session_queue 0.9997
zk_min_reads_after_write_in_session_queue 0
zk_max_reads_after_write_in_session_queue 2
zk_cnt_reads_after_write_in_session_queue 7330
zk_sum_reads_after_write_in_session_queue 7328
zk_avg_requests_in_session_queue 1.7319
zk_min_requests_in_session_queue 1
zk_max_requests_in_session_queue 6
zk_cnt_requests_in_session_queue 21988
zk_sum_requests_in_session_queue 38082
zk_avg_write_commit_proc_issued 1.0
zk_min_write_commit_proc_issued 1
zk_max_write_commit_proc_issued 1
zk_cnt_write_commit_proc_issued 7621
zk_sum_write_commit_proc_issued 7621
zk_avg_prep_process_time 0.0
zk_min_prep_process_time 0
zk_max_prep_process_time 0
zk_cnt_prep_process_time 0
zk_sum_prep_process_time 0
zk_avg_pending_session_queue_size 1.0
zk_min_pending_session_queue_size 1
zk_max_pending_session_queue_size 1
zk_cnt_pending_session_queue_size 7330
zk_sum_pending_session_queue_size 7330
zk_avg_time_waiting_empty_pool_in_commit_processor_read_ms 0.0115
zk_min_time_waiting_empty_pool_in_commit_processor_read_ms 0
zk_max_time_waiting_empty_pool_in_commit_processor_read_ms 1
zk_cnt_time_waiting_empty_pool_in_commit_processor_read_ms 7621
zk_sum_time_waiting_empty_pool_in_commit_processor_read_ms 88
zk_avg_commit_process_time 0.0197
zk_min_commit_process_time 0
zk_max_commit_process_time 11
zk_cnt_commit_process_time 181792
zk_sum_commit_process_time 3586
zk_avg_dbinittime 5338.0
zk_min_dbinittime 5338
zk_max_dbinittime 5338
zk_cnt_dbinittime 1
zk_sum_dbinittime 5338
zk_avg_netty_queued_buffer_capacity 0.0
zk_min_netty_queued_buffer_capacity 0
zk_max_netty_queued_buffer_capacity 0
zk_cnt_netty_queued_buffer_capacity 0
zk_sum_netty_queued_buffer_capacity 0
zk_avg_election_time 151.0
zk_min_election_time 20
zk_max_election_time 282
zk_cnt_election_time 2
zk_sum_election_time 302
zk_avg_commit_commit_proc_req_queued 0.0422
zk_min_commit_commit_proc_req_queued 0
zk_max_commit_commit_proc_req_queued 3
zk_cnt_commit_commit_proc_req_queued 181792
zk_sum_commit_commit_proc_req_queued 7674
zk_avg_sync_processor_batch_size 1.0374
zk_min_sync_processor_batch_size 1
zk_max_sync_processor_batch_size 3
zk_cnt_sync_processor_batch_size 294
zk_sum_sync_processor_batch_size 305
zk_avg_node_children_watch_count 3.75
zk_min_node_children_watch_count 3
zk_max_node_children_watch_count 4
zk_cnt_node_children_watch_count 4
zk_sum_node_children_watch_count 15
zk_avg_write_batch_time_in_commit_processor 0.1476
zk_min_write_batch_time_in_commit_processor 0
zk_max_write_batch_time_in_commit_processor 11
zk_cnt_write_batch_time_in_commit_processor 7621
zk_sum_write_batch_time_in_commit_processor 1125
zk_avg_read_commit_proc_issued 0.8871
zk_min_read_commit_proc_issued 0
zk_max_read_commit_proc_issued 5
zk_cnt_read_commit_proc_issued 181792
zk_sum_read_commit_proc_issued 161263
zk_avg_concurrent_request_processing_in_commit_processor 1.0207
zk_min_concurrent_request_processing_in_commit_processor 1
zk_max_concurrent_request_processing_in_commit_processor 2
zk_cnt_concurrent_request_processing_in_commit_processor 242
zk_sum_concurrent_request_processing_in_commit_processor 247
zk_avg_node_changed_watch_count 0.0
zk_min_node_changed_watch_count 0
zk_max_node_changed_watch_count 0
zk_cnt_node_changed_watch_count 0
zk_sum_node_changed_watch_count 0
zk_avg_sync_process_time 0.0918
zk_min_sync_process_time 0
zk_max_sync_process_time 1
zk_cnt_sync_process_time 305
zk_sum_sync_process_time 28
zk_avg_startup_snap_load_time 5268.0
zk_min_startup_snap_load_time 5268
zk_max_startup_snap_load_time 5268
zk_cnt_startup_snap_load_time 1
zk_sum_startup_snap_load_time 5268
zk_avg_prep_processor_queue_time_ms 0.0
zk_min_prep_processor_queue_time_ms 0
zk_max_prep_processor_queue_time_ms 0
zk_cnt_prep_processor_queue_time_ms 0
zk_sum_prep_processor_queue_time_ms 0
zk_p50_prep_processor_queue_time_ms 0
zk_p95_prep_processor_queue_time_ms 0
zk_p99_prep_processor_queue_time_ms 0
zk_p999_prep_processor_queue_time_ms 0
zk_avg_close_session_prep_time 0.0
zk_min_close_session_prep_time 0
zk_max_close_session_prep_time 0
zk_cnt_close_session_prep_time 0
zk_sum_close_session_prep_time 0
zk_p50_close_session_prep_time 0
zk_p95_close_session_prep_time 0
zk_p99_close_session_prep_time 0
zk_p999_close_session_prep_time 0
zk_avg_read_commitproc_time_ms 0.8202
zk_min_read_commitproc_time_ms 0
zk_max_read_commitproc_time_ms 13334
zk_cnt_read_commitproc_time_ms 168591
zk_sum_read_commitproc_time_ms 138271
zk_p50_read_commitproc_time_ms 0
zk_p95_read_commitproc_time_ms 1
zk_p99_read_commitproc_time_ms 1
zk_p999_read_commitproc_time_ms 2
zk_avg_updatelatency 6.0714
zk_min_updatelatency 3
zk_max_updatelatency 13
zk_cnt_updatelatency 14
zk_sum_updatelatency 85
zk_p50_updatelatency 5
zk_p95_updatelatency 13
zk_p99_updatelatency 13
zk_p999_updatelatency 13
zk_avg_local_write_committed_time_ms 0.0482
zk_min_local_write_committed_time_ms 0
zk_max_local_write_committed_time_ms 5
zk_cnt_local_write_committed_time_ms 7330
zk_sum_local_write_committed_time_ms 353
zk_p50_local_write_committed_time_ms 0
zk_p95_local_write_committed_time_ms 0
zk_p99_local_write_committed_time_ms 1
zk_p999_local_write_committed_time_ms 1
zk_avg_readlatency 1.2633
zk_min_readlatency 0
zk_max_readlatency 13334
zk_cnt_readlatency 175907
zk_sum_readlatency 222226
zk_p50_readlatency 0
zk_p95_readlatency 1
zk_p99_readlatency 1
zk_p999_readlatency 15
zk_avg_quorum_ack_latency 0.0
zk_min_quorum_ack_latency 0
zk_max_quorum_ack_latency 0
zk_cnt_quorum_ack_latency 0
zk_sum_quorum_ack_latency 0
zk_p50_quorum_ack_latency 0
zk_p95_quorum_ack_latency 0
zk_p99_quorum_ack_latency 0
zk_p999_quorum_ack_latency 0
zk_avg_om_commit_process_time_ms 0.0
zk_min_om_commit_process_time_ms 0
zk_max_om_commit_process_time_ms 0
zk_cnt_om_commit_process_time_ms 0
zk_sum_om_commit_process_time_ms 0
zk_p50_om_commit_process_time_ms 0
zk_p95_om_commit_process_time_ms 0
zk_p99_om_commit_process_time_ms 0
zk_p999_om_commit_process_time_ms 0
zk_avg_read_final_proc_time_ms 0.1748
zk_min_read_final_proc_time_ms 0
zk_max_read_final_proc_time_ms 59
zk_cnt_read_final_proc_time_ms 168591
zk_sum_read_final_proc_time_ms 29471
zk_p50_read_final_proc_time_ms 0
zk_p95_read_final_proc_time_ms 1
zk_p99_read_final_proc_time_ms 1
zk_p999_read_final_proc_time_ms 1
zk_avg_commit_propagation_latency 342.8393
zk_min_commit_propagation_latency 1
zk_max_commit_propagation_latency 103233
zk_cnt_commit_propagation_latency 305
zk_sum_commit_propagation_latency 104566
zk_p50_commit_propagation_latency 3
zk_p95_commit_propagation_latency 14
zk_p99_commit_propagation_latency 21
zk_p999_commit_propagation_latency 103233
zk_avg_dead_watchers_cleaner_latency 0.0
zk_min_dead_watchers_cleaner_latency 0
zk_max_dead_watchers_cleaner_latency 0
zk_cnt_dead_watchers_cleaner_latency 0
zk_sum_dead_watchers_cleaner_latency 0
zk_p50_dead_watchers_cleaner_latency 0
zk_p95_dead_watchers_cleaner_latency 0
zk_p99_dead_watchers_cleaner_latency 0
zk_p999_dead_watchers_cleaner_latency 0
zk_avg_write_final_proc_time_ms 0.1401
zk_min_write_final_proc_time_ms 0
zk_max_write_final_proc_time_ms 11
zk_cnt_write_final_proc_time_ms 7621
zk_sum_write_final_proc_time_ms 1068
zk_p50_write_final_proc_time_ms 0
zk_p95_write_final_proc_time_ms 1
zk_p99_write_final_proc_time_ms 1
zk_p999_write_final_proc_time_ms 1
zk_avg_proposal_ack_creation_latency 342.5574
zk_min_proposal_ack_creation_latency 1
zk_max_proposal_ack_creation_latency 103236
zk_cnt_proposal_ack_creation_latency 305
zk_sum_proposal_ack_creation_latency 104480
zk_p50_proposal_ack_creation_latency 2
zk_p95_proposal_ack_creation_latency 12
zk_p99_proposal_ack_creation_latency 91
zk_p999_proposal_ack_creation_latency 103236
zk_avg_proposal_latency 1.3849
zk_min_proposal_latency 0
zk_max_proposal_latency 18
zk_cnt_proposal_latency 304
zk_sum_proposal_latency 421
zk_p50_proposal_latency 1
zk_p95_proposal_latency 3
zk_p99_proposal_latency 10
zk_p999_proposal_latency 18
zk_avg_om_proposal_process_time_ms 0.0
zk_min_om_proposal_process_time_ms 0
zk_max_om_proposal_process_time_ms 0
zk_cnt_om_proposal_process_time_ms 0
zk_sum_om_proposal_process_time_ms 0
zk_p50_om_proposal_process_time_ms 0
zk_p95_om_proposal_process_time_ms 0
zk_p99_om_proposal_process_time_ms 0
zk_p999_om_proposal_process_time_ms 0
zk_avg_sync_processor_queue_and_flush_time_ms 2.6852
zk_min_sync_processor_queue_and_flush_time_ms 0
zk_max_sync_processor_queue_and_flush_time_ms 106
zk_cnt_sync_processor_queue_and_flush_time_ms 305
zk_sum_sync_processor_queue_and_flush_time_ms 819
zk_p50_sync_processor_queue_and_flush_time_ms 1
zk_p95_sync_processor_queue_and_flush_time_ms 9
zk_p99_sync_processor_queue_and_flush_time_ms 28
zk_p999_sync_processor_queue_and_flush_time_ms 106
zk_avg_propagation_latency 344.8125
zk_min_propagation_latency 1
zk_max_propagation_latency 103235
zk_cnt_propagation_latency 304
zk_sum_propagation_latency 104823
zk_p50_propagation_latency 4
zk_p95_propagation_latency 15
zk_p99_propagation_latency 22
zk_p999_propagation_latency 103235
zk_avg_server_write_committed_time_ms 0.3058
zk_min_server_write_committed_time_ms 0
zk_max_server_write_committed_time_ms 10
zk_cnt_server_write_committed_time_ms 291
zk_sum_server_write_committed_time_ms 89
zk_p50_server_write_committed_time_ms 0
zk_p95_server_write_committed_time_ms 1
zk_p99_server_write_committed_time_ms 8
zk_p999_server_write_committed_time_ms 10
zk_avg_sync_processor_queue_time_ms 0.3443
zk_min_sync_processor_queue_time_ms 0
zk_max_sync_processor_queue_time_ms 11
zk_cnt_sync_processor_queue_time_ms 305
zk_sum_sync_processor_queue_time_ms 105
zk_p50_sync_processor_queue_time_ms 0
zk_p95_sync_processor_queue_time_ms 2
zk_p99_sync_processor_queue_time_ms 7
zk_p999_sync_processor_queue_time_ms 11
zk_avg_sync_processor_queue_flush_time_ms 2.2245
zk_min_sync_processor_queue_flush_time_ms 0
zk_max_sync_processor_queue_flush_time_ms 106
zk_cnt_sync_processor_queue_flush_time_ms 294
zk_sum_sync_processor_queue_flush_time_ms 654
zk_p50_sync_processor_queue_flush_time_ms 1
zk_p95_sync_processor_queue_flush_time_ms 4
zk_p99_sync_processor_queue_flush_time_ms 31
zk_p999_sync_processor_queue_flush_time_ms 106
zk_avg_write_commitproc_time_ms 0.5658
zk_min_write_commitproc_time_ms 0
zk_max_write_commitproc_time_ms 20
zk_cnt_write_commitproc_time_ms 7330
zk_sum_write_commitproc_time_ms 4147
zk_p50_write_commitproc_time_ms 1
zk_p95_write_commitproc_time_ms 1
zk_p99_write_commitproc_time_ms 2
zk_p999_write_commitproc_time_ms 4
zk_avg_trufflesolr_read_per_namespace 303.6124
zk_min_trufflesolr_read_per_namespace 80
zk_max_trufflesolr_read_per_namespace 3361
zk_cnt_trufflesolr_read_per_namespace 22339
zk_sum_trufflesolr_read_per_namespace 6782397
zk_avg_trufflesolr_write_per_namespace 347.3285
zk_min_trufflesolr_write_per_namespace 34
zk_max_trufflesolr_write_per_namespace 1323
zk_cnt_trufflesolr_write_per_namespace 277
zk_sum_trufflesolr_write_per_namespace 96210
zk_avg_zookeeper_write_per_namespace 182.0
zk_min_zookeeper_write_per_namespace 182
zk_max_zookeeper_write_per_namespace 182
zk_cnt_zookeeper_write_per_namespace 3
zk_sum_zookeeper_write_per_namespace 546
{code}
> Solr 8.6 / ZK 3.6.1 / Admin UI - ZK Status Null
> -----------------------------------------------
>
> Key: SOLR-14669
> URL: https://issues.apache.org/jira/browse/SOLR-14669
> Project: Solr
> Issue Type: Bug
> Security Level: Public(Default Security Level. Issues are Public)
> Components: Admin UI, SolrCloud
> Affects Versions: 8.6
> Reporter: Jörn Franke
> Priority: Minor
>
> When opening the Admin UI / ZK Status page it shows just null. Solr 8.6.0 /
> ZK 3.6.1. Zk is a 3 node ensemble.
> It seems to be cosmetic in the UI - otherwise Solr seems to work fine.
> Deleted already browser cache and restarted browser. Issue persists.
> In the logfiles I find the following error:
> 2020-07-20 16:34:27.853 ERROR (qtp767511741-2227) [ ]
> o.a.s.h.RequestHandlerBase java.lang.NumberFormatException: For input string:
> "null"
> at
> java.base/java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
> at java.base/java.lang.Integer.parseInt(Integer.java:652)
> at java.base/java.lang.Integer.parseInt(Integer.java:770)
> at
> org.apache.solr.handler.admin.ZookeeperStatusHandler.getZkStatus(ZookeeperStatusHandler.java:116)
> at
> org.apache.solr.handler.admin.ZookeeperStatusHandler.handleRequestBody(ZookeeperStatusHandler.java:78)
> at
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:211)
> at
> org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:839)
> at
> org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:805)
> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:558)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:419)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
> at
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1602)
> at
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
> at
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
> at
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1711)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1347)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1678)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1249)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> at
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
> at
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:152)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at
> org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at org.eclipse.jetty.server.Server.handle(Server.java:505)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> at
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at
> org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:427)
> at
> org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:321)
> at
> org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
> at
> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
> at java.base/java.lang.Thread.run(Thread.java:834)
> 2020-07-20 16:34:27.853 INFO (qtp767511741-2227) [ ] o.a.s.s.HttpSolrCall
> [admin] webapp=null path=/admin/zookeeper/status
> params=\{wt=json&_=1595262857191} status=500 QTime=28
> 2020-07-20 16:34:27.853 ERROR (qtp767511741-2227) [ ] o.a.s.s.HttpSolrCall
> null:java.lang.NumberFormatException: For input string: "null"
> at
> java.base/java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
> at java.base/java.lang.Integer.parseInt(Integer.java:652)
> at java.base/java.lang.Integer.parseInt(Integer.java:770)
> at
> org.apache.solr.handler.admin.ZookeeperStatusHandler.getZkStatus(ZookeeperStatusHandler.java:116)
> at
> org.apache.solr.handler.admin.ZookeeperStatusHandler.handleRequestBody(ZookeeperStatusHandler.java:78)
> at
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:211)
> at
> org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:839)
> at
> org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:805)
> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:558)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:419)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
> at
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1602)
> at
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
> at
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
> at
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1711)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1347)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1678)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1249)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> at
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
> at
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:152)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at
> org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at org.eclipse.jetty.server.Server.handle(Server.java:505)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> at
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at
> org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:427)
> at
> org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:321)
> at
> org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
> at
> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
> at java.base/java.lang.Thread.run(Thread.java:834)
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
